From 73e75e33eed6f7194314a3cd5b9a9831ea536e1e Mon Sep 17 00:00:00 2001 From: sitay Date: Mon, 7 Aug 2023 15:32:21 +0300 Subject: [PATCH 01/13] initial commit --- .../plugins/spotinst/api/SpotinstApi.java | 41 ++++++++++++++ .../spotinst/cloud/AwsSpotinstCloud.java | 55 ++++++++++++++++++- .../spotinst/cloud/AzureSpotCloud.java | 20 +++++-- .../spotinst/cloud/AzureSpotinstCloud.java | 16 +++++- .../spotinst/cloud/BaseSpotinstCloud.java | 31 +++++++++-- .../spotinst/cloud/GcpSpotinstCloud.java | 16 +++++- .../model/aws/AwsStatefulInstance.java | 34 ++++++++++++ .../aws/AwsStatefulInstancesResponse.java | 6 ++ .../plugins/spotinst/repos/AwsGroupRepo.java | 35 ++++++++++++ .../plugins/spotinst/repos/IAwsGroupRepo.java | 5 ++ .../plugins/spotinst/slave/SpotinstSlave.java | 4 +- 11 files changed, 241 insertions(+), 22 deletions(-) create mode 100644 src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstance.java create mode 100644 src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesResponse.java diff --git a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java index 0517e02e..32536b06 100644 --- a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java +++ b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java @@ -89,6 +89,26 @@ public static List getAwsGroupInstances(String groupId, String return retVal; } + public static List getAwsStatefulInstances(String groupId, + String accountId) throws ApiException { + List retVal = new LinkedList<>(); + Map headers = buildHeaders(); + Map queryParams = buildQueryParams(accountId); + + RestResponse response = + RestClient.sendGet(SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/statefulInstance", headers, + queryParams); + + AwsStatefulInstancesResponse instancesResponse = + getCastedResponse(response, AwsStatefulInstancesResponse.class); + + if (instancesResponse.getResponse().getItems().size() > 0) { + retVal = instancesResponse.getResponse().getItems(); + } + + return retVal; + } + public static AwsScaleUpResult awsScaleUp(String groupId, int adjustment, String accountId) throws ApiException { AwsScaleUpResult retVal = null; Map headers = buildHeaders(); @@ -128,6 +148,27 @@ public static Boolean awsDetachInstance(String instanceId, String accountId) thr return retVal; } + public static Boolean awsDeallocateInstance(String groupId, String statefulInstanceId, + String accountId) throws ApiException { + Map headers = buildHeaders(); + Map queryParams = buildQueryParams(accountId); + + AwsDetachInstancesRequest request = new AwsDetachInstancesRequest(); + request.setShouldDecrementTargetCapacity(true); + request.setShouldTerminateInstances(true); + + String body = JsonMapper.toJson(request); + + RestResponse response = RestClient.sendPut( + SPOTINST_API_HOST + "/aws/ec2/group/" + groupId + "/statefulInstance/" + statefulInstanceId + + "/deallocate", body, headers, queryParams); + + getCastedResponse(response, ApiEmptyResponse.class); + Boolean retVal = true; + + return retVal; + } + public static List getAllAwsInstanceTypes(String accountId) throws ApiException { List retVal; Map headers = buildHeaders(); diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java index 6ece8bde..5a7d2705 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java @@ -14,6 +14,7 @@ import hudson.slaves.EnvironmentVariablesNodeProperty; import hudson.tools.ToolLocationNodeProperty; import jenkins.model.Jenkins; +import org.apache.commons.collections.CollectionUtils; import org.kohsuke.stapler.DataBoundConstructor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -99,9 +100,57 @@ List scaleUp(ProvisionRequest request) { return retVal; } + @Override + protected String getStatefulInstanceId(String instanceId) { + String retVal = null; + + IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); + ApiResponse> statefulInstancesResponse = + awsGroupRepo.getStatefulInstances(groupId, accountId); + + if (statefulInstancesResponse.isRequestSucceed()) { + List groupStatefulInstances = statefulInstancesResponse.getValue(); + boolean isGroupHasStatefulInstances = CollectionUtils.isNotEmpty(groupStatefulInstances); + + if (isGroupHasStatefulInstances) { + Optional optionalAwsStatefulInstance = groupStatefulInstances.stream() + .filter(statefulInstance -> instanceId.equals( + statefulInstance.getInstanceId())) + .findFirst(); + boolean isStatefulInstance = optionalAwsStatefulInstance.isPresent(); + + if (isStatefulInstance) { + retVal = optionalAwsStatefulInstance.get().getId(); + } + } + } + + return retVal; + } + + @Override + public Boolean deallocateInstance(String statefulInstanceId) { + boolean retVal = false; + + IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); + ApiResponse detachInstanceResponse = + awsGroupRepo.deallocateInstance(groupId, statefulInstanceId, accountId); + + if (detachInstanceResponse.isRequestSucceed()) { + LOGGER.info(String.format("Stateful Instance %s deallocated", statefulInstanceId)); + retVal = true; + } + else { + LOGGER.error(String.format("Failed to deallocate instance %s. Errors: %s", statefulInstanceId, + detachInstanceResponse.getErrors())); + } + + return retVal; + } + @Override public Boolean detachInstance(String instanceId) { - Boolean retVal = false; + boolean retVal = false; IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); ApiResponse detachInstanceResponse = awsGroupRepo.detachInstance(instanceId, this.accountId); @@ -326,7 +375,7 @@ private List getGroupInstanceAndSpotIds(List elastigro } private Boolean isSlaveExistForInstance(AwsGroupInstance instance) { - Boolean retVal = false; + boolean retVal = false; Node node; String instanceId = instance.getInstanceId(); @@ -381,7 +430,7 @@ private void initExecutorsByInstanceType() { this.executorsByInstanceType.put(type, executors); if (instance.getIsValid() == false) { - LOGGER.error(String.format("Invalid type \'%s\' in group \'%s\'", type, this.getGroupId())); + LOGGER.error(String.format("Invalid type '%s' in group '%s'", type, this.getGroupId())); invalidInstanceTypes.add(type); } } diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java index 4e03cba2..c769c526 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java @@ -85,10 +85,20 @@ List scaleUp(ProvisionRequest request) { return retVal; } + @Override + protected String getStatefulInstanceId(String instanceId) { + return null;//TODO: implement + } + + @Override + public Boolean deallocateInstance(String instanceId) { + return false;//TODO: implement + } + @Override public Boolean detachInstance(String instanceId) { - Boolean retVal = false; - IAzureVmGroupRepo azureVmGroupRepo = RepoManager.getInstance().getAzureVmGroupRepo(); + boolean retVal = false; + IAzureVmGroupRepo azureVmGroupRepo = RepoManager.getInstance().getAzureVmGroupRepo(); ApiResponse detachVmResponse = azureVmGroupRepo.detachVM(groupId, instanceId, this.accountId); if (detachVmResponse.isRequestSucceed()) { @@ -212,7 +222,7 @@ private void removeOldSlaveInstances(List azureGroupVms) { for (SpotinstSlave slave : allGroupsSlaves) { String slaveInstanceId = slave.getInstanceId(); - Boolean slaveIdNotNull = slaveInstanceId != null; + boolean slaveIdNotNull = slaveInstanceId != null; Boolean slaveExistsInEg = elastigroupVmIds.contains(slaveInstanceId); if (slaveIdNotNull && BooleanUtils.isFalse(slaveExistsInEg)) { @@ -241,7 +251,7 @@ private void addNewSlaveInstances(List azureGroupVms) { if (azureGroupVms.size() > 0) { for (AzureGroupVm vm : azureGroupVms) { - Boolean doesSlaveNotExist = BooleanUtils.isFalse(isSlaveExistForInstance(vm)); + boolean doesSlaveNotExist = BooleanUtils.isFalse(isSlaveExistForInstance(vm)); if (doesSlaveNotExist) { LOGGER.info(String.format("Instance: %s of group: %s doesn't have slave , adding new one", @@ -279,7 +289,7 @@ private void addSpotinstSlave(AzureGroupVm vm) { private Boolean isSlaveExistForInstance(AzureGroupVm vm) { - Boolean retVal = false; + boolean retVal = false; Node node = Jenkins.get().getNode(vm.getVmName()); if (node != null) { diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java index e15de293..0af037b4 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java @@ -75,10 +75,20 @@ List scaleUp(ProvisionRequest request) { return retVal; } + @Override + protected String getStatefulInstanceId(String instanceId) { + return null;//TODO: implement + } + + @Override + public Boolean deallocateInstance(String instanceId){ + return false;//TODO: implement + } + @Override public Boolean detachInstance(String instanceId) { - Boolean retVal = false; - IAzureGroupRepo azureGroupRepo = RepoManager.getInstance().getAzureGroupRepo(); + boolean retVal = false; + IAzureGroupRepo azureGroupRepo = RepoManager.getInstance().getAzureGroupRepo(); ApiResponse detachInstanceResponse = azureGroupRepo.detachInstance(groupId, instanceId, accountId); if (detachInstanceResponse.isRequestSucceed()) { @@ -281,7 +291,7 @@ private List getGroupInstanceIds(List azureGroupInst } private Boolean isSlaveExistForInstance(AzureGroupInstance instance) { - Boolean retVal = false; + boolean retVal = false; Node node = Jenkins.getInstance().getNode(instance.getInstanceId()); diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index 48557f2e..a9af5bb6 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -356,10 +356,10 @@ protected void terminateOfflineSlaves(SpotinstSlave slave, String slaveInstanceI SlaveComputer computer = slave.getComputer(); if (computer != null) { Integer offlineThreshold = getSlaveOfflineThreshold(); - Boolean isSlaveConnecting = computer.isConnecting(); - Boolean isSlaveOffline = computer.isOffline(); + boolean isSlaveConnecting = computer.isConnecting(); + boolean isSlaveOffline = computer.isOffline(); // the computer is actively marked as temporary offline - Boolean temporarilyOffline = computer.isTemporarilyOffline(); + boolean temporarilyOffline = computer.isTemporarilyOffline(); long idleStartMillis = computer.getIdleStartMilliseconds(); long idleInMillis = System.currentTimeMillis() - idleStartMillis; @@ -519,7 +519,7 @@ protected SpotinstSlave buildSpotinstSlave(String id, String instanceType, Strin private ComputerLauncher buildLauncherForAgent(String instanceId) throws IOException { ComputerLauncher retVal; - Boolean isSshCloud = this.getConnectionMethod().equals(ConnectionMethodEnum.SSH); + boolean isSshCloud = this.getConnectionMethod().equals(ConnectionMethodEnum.SSH); SlaveInstanceDetails instanceDetailsById = getSlaveDetails(instanceId); if (isSshCloud) { @@ -633,7 +633,7 @@ protected PendingExecutorsCounts getPendingExecutors(ProvisionRequest request) { } protected Integer getNumOfExecutors(String instanceType) { - Integer retVal; + int retVal; boolean isSingleTaskNodesEnabled = getIsSingleTaskNodesEnabled(); if (isSingleTaskNodesEnabled) { @@ -853,7 +853,26 @@ public void setIsSingleTaskNodesEnabled(Boolean isSingleTaskNodesEnabled) { //region Abstract Methods abstract List scaleUp(ProvisionRequest request); - public abstract Boolean detachInstance(String instanceId); + public Boolean removeInstance(String instanceId) { + boolean retVal; + String statefulInstanceId = getStatefulInstanceId(instanceId); + boolean isStateful = statefulInstanceId != null; + + if (isStateful) { + retVal = deallocateInstance(statefulInstanceId); + } + else { + retVal = detachInstance(instanceId); + } + + return retVal; + } + + protected abstract String getStatefulInstanceId(String instanceId); + + protected abstract Boolean detachInstance(String instanceId); + + protected abstract Boolean deallocateInstance(String instanceId); public abstract String getCloudUrl(); diff --git a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java index d7835e71..6a7c0722 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java @@ -98,10 +98,20 @@ List scaleUp(ProvisionRequest request) { return retVal; } + @Override + protected String getStatefulInstanceId(String instanceId) { + return null;//TODO: implement + } + + @Override + public Boolean deallocateInstance(String instanceId){ + return false;//TODO: implement + } + @Override public Boolean detachInstance(String instanceId) { - Boolean retVal = false; - IGcpGroupRepo gcpGroupRepo = RepoManager.getInstance().getGcpGroupRepo(); + boolean retVal = false; + IGcpGroupRepo gcpGroupRepo = RepoManager.getInstance().getGcpGroupRepo(); ApiResponse detachInstanceResponse = gcpGroupRepo.detachInstance(groupId, instanceId, this.accountId); if (detachInstanceResponse.isRequestSucceed()) { @@ -263,7 +273,7 @@ private List getGcpInstanceNames(List gcpGroupInstance } private Boolean isSlaveExistForGcpInstance(GcpGroupInstance instance) { - Boolean retVal = false; + boolean retVal = false; Node node = Jenkins.getInstance().getNode(instance.getInstanceName()); diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstance.java b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstance.java new file mode 100644 index 00000000..f5ef9521 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstance.java @@ -0,0 +1,34 @@ +package hudson.plugins.spotinst.model.aws; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * Created by ItayShklar on 07/08/2023. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class AwsStatefulInstance { + //region members + private String id; + private String instanceId; + //endregion + + //region getters & setters + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getInstanceId() { + return instanceId; + } + + public void setInstanceId(String instanceId) { + this.instanceId = instanceId; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesResponse.java b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesResponse.java new file mode 100644 index 00000000..af564037 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesResponse.java @@ -0,0 +1,6 @@ +package hudson.plugins.spotinst.model.aws; + +import hudson.plugins.spotinst.api.infra.BaseItemsResponse; + +public class AwsStatefulInstancesResponse extends BaseItemsResponse { +} diff --git a/src/main/java/hudson/plugins/spotinst/repos/AwsGroupRepo.java b/src/main/java/hudson/plugins/spotinst/repos/AwsGroupRepo.java index 59d23c95..03a6a730 100644 --- a/src/main/java/hudson/plugins/spotinst/repos/AwsGroupRepo.java +++ b/src/main/java/hudson/plugins/spotinst/repos/AwsGroupRepo.java @@ -7,6 +7,7 @@ import hudson.plugins.spotinst.model.aws.AwsInstanceType; import hudson.plugins.spotinst.model.aws.AwsGroupInstance; import hudson.plugins.spotinst.model.aws.AwsScaleUpResult; +import hudson.plugins.spotinst.model.aws.AwsStatefulInstance; import java.util.List; @@ -32,6 +33,23 @@ public ApiResponse> getGroupInstances(String groupId, Str return retVal; } + @Override + public ApiResponse> getStatefulInstances(String groupId, String accountId) { + ApiResponse> retVal; + + try { + List instances = SpotinstApi.getAwsStatefulInstances(groupId, accountId); + + retVal = new ApiResponse<>(instances); + + } + catch (ApiException e) { + retVal = ExceptionHelper.handleDalException(e); + } + + return retVal; + } + @Override public ApiResponse detachInstance(String instanceId, String accountId) { ApiResponse retVal; @@ -49,6 +67,23 @@ public ApiResponse detachInstance(String instanceId, String accountId) return retVal; } + @Override + public ApiResponse deallocateInstance(String groupId, String statefulInstanceId, String accountId) { + ApiResponse retVal; + + try { + Boolean isDetached = SpotinstApi.awsDeallocateInstance(groupId, statefulInstanceId, accountId); + + retVal = new ApiResponse<>(isDetached); + + } + catch (ApiException e) { + retVal = ExceptionHelper.handleDalException(e); + } + + return retVal; + } + @Override public ApiResponse scaleUp(String groupId, Integer adjustment, String accountId) { ApiResponse retVal; diff --git a/src/main/java/hudson/plugins/spotinst/repos/IAwsGroupRepo.java b/src/main/java/hudson/plugins/spotinst/repos/IAwsGroupRepo.java index 1f54eb91..0451eded 100644 --- a/src/main/java/hudson/plugins/spotinst/repos/IAwsGroupRepo.java +++ b/src/main/java/hudson/plugins/spotinst/repos/IAwsGroupRepo.java @@ -4,6 +4,7 @@ import hudson.plugins.spotinst.model.aws.AwsInstanceType; import hudson.plugins.spotinst.model.aws.AwsGroupInstance; import hudson.plugins.spotinst.model.aws.AwsScaleUpResult; +import hudson.plugins.spotinst.model.aws.AwsStatefulInstance; import java.util.List; @@ -13,8 +14,12 @@ public interface IAwsGroupRepo { ApiResponse> getGroupInstances(String groupId, String accountId); + ApiResponse> getStatefulInstances(String groupId, String accountId); + ApiResponse detachInstance(String instanceId, String accountId); + ApiResponse deallocateInstance(String groupId, String statefulInstanceId, String accountId); + ApiResponse scaleUp(String groupId, Integer adjustment, String accountId); ApiResponse> getAllInstanceTypes(String accountId); diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java index 62ae7951..6e71c91d 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java @@ -177,7 +177,7 @@ public void terminate() { boolean isGroupManagedByThisController = getSpotinstCloud().isCloudReadyForGroupCommunication(); if (isGroupManagedByThisController) { - Boolean isTerminated = getSpotinstCloud().detachInstance(instanceId); + Boolean isTerminated = getSpotinstCloud().removeInstance(instanceId); if (isTerminated) { LOGGER.info(String.format("Instance: %s terminated successfully", getInstanceId())); @@ -204,7 +204,7 @@ public Boolean forceTerminate() { boolean isGroupManagedByThisController = getSpotinstCloud().isCloudReadyForGroupCommunication(); if (isGroupManagedByThisController) { - Boolean isTerminated = getSpotinstCloud().detachInstance(instanceId); + Boolean isTerminated = getSpotinstCloud().removeInstance(instanceId); if (isTerminated) { LOGGER.info(String.format("Instance: %s terminated successfully", getInstanceId())); From d636c394a91a0b39eb3c43da9934b918ad5e8f27 Mon Sep 17 00:00:00 2001 From: sitay Date: Sun, 13 Aug 2023 15:13:11 +0300 Subject: [PATCH 02/13] initial commit --- .../spotinst/cloud/AwsSpotinstCloud.java | 20 +++++++++++++++++-- .../spotinst/cloud/BaseSpotinstCloud.java | 4 +++- .../spotinst/common/GroupLockingManager.java | 6 ++++-- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java index 5a7d2705..c3962386 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java @@ -169,8 +169,9 @@ public Boolean detachInstance(String instanceId) { @Override protected void internalSyncGroupInstances() { - IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); - ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); + IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); + ApiResponse> instancesResponse = + awsGroupRepo.getGroupInstances(groupId, this.accountId); if (instancesResponse.isRequestSucceed()) { List instances = instancesResponse.getValue(); @@ -178,6 +179,21 @@ protected void internalSyncGroupInstances() { Map slaveInstancesDetailsByInstanceId = new HashMap<>(); + List statefulInstances; + ApiResponse> statefulInstancesResponse = + awsGroupRepo.getStatefulInstances(groupId, this.accountId); + + if(statefulInstancesResponse.isRequestSucceed()){ + statefulInstances = statefulInstancesResponse.getValue(); + } + else{ + statefulInstances = new LinkedList<>(); + } + + Map ssiByInstanceId = new HashMap<>(); + statefulInstances.forEach(ssi -> ssiByInstanceId.put(ssi.getInstanceId(), ssi)); + this.ssiByInstanceId = ssiByInstanceId; + for (AwsGroupInstance instance : instances) { SlaveInstanceDetails instanceDetails = SlaveInstanceDetails.build(instance); slaveInstancesDetailsByInstanceId.put(instanceDetails.getInstanceId(), instanceDetails); diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index a9af5bb6..6e05903f 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -5,6 +5,7 @@ import hudson.model.labels.LabelAtom; import hudson.plugins.spotinst.api.infra.JsonMapper; import hudson.plugins.spotinst.common.*; +import hudson.plugins.spotinst.model.aws.AwsStatefulInstance; import hudson.plugins.spotinst.slave.*; import hudson.plugins.sshslaves.SSHConnector; import hudson.slaves.*; @@ -40,6 +41,7 @@ public abstract class BaseSpotinstCloud extends Cloud { protected String groupId; protected Map pendingInstances; protected Map slaveInstancesDetailsByInstanceId; + protected Map ssiByInstanceId; private String labelString; private String idleTerminationMinutes; private String workspaceDir; @@ -856,7 +858,7 @@ public void setIsSingleTaskNodesEnabled(Boolean isSingleTaskNodesEnabled) { public Boolean removeInstance(String instanceId) { boolean retVal; String statefulInstanceId = getStatefulInstanceId(instanceId); - boolean isStateful = statefulInstanceId != null; + boolean isStateful = statefulInstanceId != null; if (isStateful) { retVal = deallocateInstance(statefulInstanceId); diff --git a/src/main/java/hudson/plugins/spotinst/common/GroupLockingManager.java b/src/main/java/hudson/plugins/spotinst/common/GroupLockingManager.java index b2ad61a0..9ed0856e 100644 --- a/src/main/java/hudson/plugins/spotinst/common/GroupLockingManager.java +++ b/src/main/java/hudson/plugins/spotinst/common/GroupLockingManager.java @@ -261,8 +261,9 @@ private void handleInitializingFailureTimeout(String errorDescription) { TimeUtils.isTimePassedInSeconds(initializingStateStartTimeStamp, INITIALIZING_PERIOD_IN_SECONDS); if (isTimeout) { - LOGGER.error("Initialization time has expired, error description: {}", errorDescription); - setFailedState(errorDescription); + String timeoutErrorDescription = + String.format("Initialization time has expired, error description: %s", errorDescription); + setFailedState(timeoutErrorDescription); } else { LOGGER.warn( @@ -282,6 +283,7 @@ private void setReadyState() { } public void setFailedState(String description) { + LOGGER.error("Cloud failed to communicate with the group. {}", description); setCloudCommunicationState(SpotinstCloudCommunicationState.FAILED); setErrorDescription(description); } From 678f14f625fb870e44bd24684b1dfa83d0499882 Mon Sep 17 00:00:00 2001 From: sitay Date: Mon, 14 Aug 2023 17:56:46 +0300 Subject: [PATCH 03/13] initial commit --- .../spotinst/jobs/MyBuildListener.java | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 src/main/java/hudson/plugins/spotinst/jobs/MyBuildListener.java diff --git a/src/main/java/hudson/plugins/spotinst/jobs/MyBuildListener.java b/src/main/java/hudson/plugins/spotinst/jobs/MyBuildListener.java new file mode 100644 index 00000000..64462d86 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/jobs/MyBuildListener.java @@ -0,0 +1,33 @@ +//package hudson.plugins.spotinst.jobs; +// +//import hudson.Extension; +//import hudson.model.AbstractBuild; +//import hudson.model.Node; +//import hudson.model.Result; +//import hudson.model.TaskListener; +//import hudson.model.listeners.RunListener; +//import hudson.plugins.spotinst.slave.SpotinstSlave; +// +//@Extension +//public class MyBuildListener extends RunListener { +// +// @Override +// public void onCompleted(AbstractBuild build, TaskListener listener) { +// super.onCompleted(build, listener); +// +// if (build.getResult() == Result.FAILURE) { +// // Build failed, check if instance alive. +// // if not +// Node node = build.getBuiltOn(); +// +// if(node instanceof SpotinstSlave){ +// SpotinstSlave spotinstSlave = (SpotinstSlave) node; +// String instanceId = spotinstSlave.getInstanceId(); +// +// if(instanceId != null){ +// +// } +// } +// } +// } +//} From f36c6f0c5ac0c1f97ddc7ae82b1bfb5cfbbaaa69 Mon Sep 17 00:00:00 2001 From: sitay Date: Thu, 17 Aug 2023 18:28:25 +0300 Subject: [PATCH 04/13] initial commit --- .../spotinst/jobs/MyBuildListener.java | 33 ------------------- 1 file changed, 33 deletions(-) delete mode 100644 src/main/java/hudson/plugins/spotinst/jobs/MyBuildListener.java diff --git a/src/main/java/hudson/plugins/spotinst/jobs/MyBuildListener.java b/src/main/java/hudson/plugins/spotinst/jobs/MyBuildListener.java deleted file mode 100644 index 64462d86..00000000 --- a/src/main/java/hudson/plugins/spotinst/jobs/MyBuildListener.java +++ /dev/null @@ -1,33 +0,0 @@ -//package hudson.plugins.spotinst.jobs; -// -//import hudson.Extension; -//import hudson.model.AbstractBuild; -//import hudson.model.Node; -//import hudson.model.Result; -//import hudson.model.TaskListener; -//import hudson.model.listeners.RunListener; -//import hudson.plugins.spotinst.slave.SpotinstSlave; -// -//@Extension -//public class MyBuildListener extends RunListener { -// -// @Override -// public void onCompleted(AbstractBuild build, TaskListener listener) { -// super.onCompleted(build, listener); -// -// if (build.getResult() == Result.FAILURE) { -// // Build failed, check if instance alive. -// // if not -// Node node = build.getBuiltOn(); -// -// if(node instanceof SpotinstSlave){ -// SpotinstSlave spotinstSlave = (SpotinstSlave) node; -// String instanceId = spotinstSlave.getInstanceId(); -// -// if(instanceId != null){ -// -// } -// } -// } -// } -//} From e1f6892bb6c6117331e131875654f5f2a59ead04 Mon Sep 17 00:00:00 2001 From: sitay Date: Wed, 30 Aug 2023 12:31:12 +0300 Subject: [PATCH 05/13] initial commit --- .../spotinst/cloud/AwsSpotinstCloud.java | 71 ++++++++++------ .../spotinst/cloud/AzureSpotCloud.java | 14 ++-- .../spotinst/cloud/AzureSpotinstCloud.java | 12 +-- .../spotinst/cloud/BaseSpotinstCloud.java | 56 +++++++++---- .../spotinst/cloud/GcpSpotinstCloud.java | 4 +- .../spotinst/cloud/SpotReTriggerBuilds.java | 68 ++++++++++++++++ .../queue/SpotQueueTaskDispatcher.java | 65 +++++++++++++++ .../spotinst/queue/SsiByTaskMapper.java | 31 +++++++ .../queue/StatefulInterruptedTask.java | 74 +++++++++++++++++ .../queue/StatefulQueueDecisionHandler.java | 44 ++++++++++ .../spotinst/slave/SpotLauncherHelper.java | 22 +++-- .../spotinst/slave/SpotinstComputer.java | 42 ++++++++-- .../plugins/spotinst/slave/SpotinstSlave.java | 16 +++- .../cloud/BaseSpotinstCloud/config.jelly | 16 ++-- .../cloud/SpotReTriggerBuilds/config.jelly | 10 +++ .../plugins/spotinst/SpotinstCloudTest.java | 81 ++++++++++++------- 16 files changed, 515 insertions(+), 111 deletions(-) create mode 100644 src/main/java/hudson/plugins/spotinst/cloud/SpotReTriggerBuilds.java create mode 100644 src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java create mode 100644 src/main/java/hudson/plugins/spotinst/queue/SsiByTaskMapper.java create mode 100644 src/main/java/hudson/plugins/spotinst/queue/StatefulInterruptedTask.java create mode 100644 src/main/java/hudson/plugins/spotinst/queue/StatefulQueueDecisionHandler.java create mode 100644 src/main/resources/hudson/plugins/spotinst/cloud/SpotReTriggerBuilds/config.jelly diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java index c3962386..168aa221 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java @@ -41,7 +41,7 @@ public class AwsSpotinstCloud extends BaseSpotinstCloud { @DataBoundConstructor public AwsSpotinstCloud(String groupId, String labelString, String idleTerminationMinutes, String workspaceDir, List executorsForTypes, SlaveUsageEnum usage, - String tunnel, Boolean shouldUseWebsocket, Boolean shouldRetriggerBuilds, String vmargs, + String tunnel, Boolean shouldUseWebsocket, SpotReTriggerBuilds spotReTriggerBuilds, String vmargs, EnvironmentVariablesNodeProperty environmentVariables, ToolLocationNodeProperty toolLocations, String accountId, ConnectionMethodEnum connectionMethod, ComputerConnector computerConnector, @@ -49,7 +49,7 @@ public AwsSpotinstCloud(String groupId, String labelString, String idleTerminati Integer pendingThreshold) { super(groupId, labelString, idleTerminationMinutes, workspaceDir, usage, tunnel, shouldUseWebsocket, - shouldRetriggerBuilds, vmargs, environmentVariables, toolLocations, accountId, connectionMethod, + spotReTriggerBuilds, vmargs, environmentVariables, toolLocations, accountId, connectionMethod, computerConnector, shouldUsePrivateIp, globalExecutorOverride, pendingThreshold); this.executorsForTypes = new LinkedList<>(); @@ -169,9 +169,8 @@ public Boolean detachInstance(String instanceId) { @Override protected void internalSyncGroupInstances() { - IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); - ApiResponse> instancesResponse = - awsGroupRepo.getGroupInstances(groupId, this.accountId); + IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); + ApiResponse> instancesResponse = awsGroupRepo.getGroupInstances(groupId, this.accountId); if (instancesResponse.isRequestSucceed()) { List instances = instancesResponse.getValue(); @@ -179,21 +178,6 @@ protected void internalSyncGroupInstances() { Map slaveInstancesDetailsByInstanceId = new HashMap<>(); - List statefulInstances; - ApiResponse> statefulInstancesResponse = - awsGroupRepo.getStatefulInstances(groupId, this.accountId); - - if(statefulInstancesResponse.isRequestSucceed()){ - statefulInstances = statefulInstancesResponse.getValue(); - } - else{ - statefulInstances = new LinkedList<>(); - } - - Map ssiByInstanceId = new HashMap<>(); - statefulInstances.forEach(ssi -> ssiByInstanceId.put(ssi.getInstanceId(), ssi)); - this.ssiByInstanceId = ssiByInstanceId; - for (AwsGroupInstance instance : instances) { SlaveInstanceDetails instanceDetails = SlaveInstanceDetails.build(instance); slaveInstancesDetailsByInstanceId.put(instanceDetails.getInstanceId(), instanceDetails); @@ -201,6 +185,8 @@ protected void internalSyncGroupInstances() { this.slaveInstancesDetailsByInstanceId = new HashMap<>(slaveInstancesDetailsByInstanceId); + syncGroupStatefulInstances(); + addNewSlaveInstances(instances); removeOldSlaveInstances(instances); } @@ -279,14 +265,32 @@ protected int getOverriddenNumberOfExecutors(String instanceType) { return retVal; } + private void syncGroupStatefulInstances() { + List statefulInstances; + IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); + ApiResponse> statefulInstancesResponse = + awsGroupRepo.getStatefulInstances(groupId, this.accountId); + + if (statefulInstancesResponse.isRequestSucceed()) { + statefulInstances = statefulInstancesResponse.getValue(); + } + else { + statefulInstances = new LinkedList<>(); + } + + Map ssiById = new HashMap<>(); + statefulInstances.forEach(ssi -> ssiById.put(ssi.getId(), ssi)); + this.ssiById = ssiById; + } + private List handleNewAwsSpots(AwsScaleUpResult scaleUpResult, String label) { List retVal = new LinkedList<>(); LOGGER.info(String.format("%s new spot requests created", scaleUpResult.getNewSpotRequests().size())); for (AwsScaleResultNewSpot spot : scaleUpResult.getNewSpotRequests()) { - - SpotinstSlave slave = handleNewAwsInstance(spot.getInstanceId(), spot.getInstanceType(), label); + String ssiId = getSsiByInstance(spot.getInstanceId()); + SpotinstSlave slave = handleNewAwsInstance(spot.getInstanceId(), spot.getInstanceType(), ssiId, label); retVal.add(slave); } @@ -300,17 +304,19 @@ private List handleNewAwsInstances(AwsScaleUpResult scaleUpResult LOGGER.info(String.format("%s new instances launched", scaleUpResult.getNewInstances().size())); for (AwsScaleResultNewInstance instance : scaleUpResult.getNewInstances()) { - SpotinstSlave slave = handleNewAwsInstance(instance.getInstanceId(), instance.getInstanceType(), label); + String ssiId = getSsiByInstance(instance.getInstanceId()); + SpotinstSlave slave = + handleNewAwsInstance(instance.getInstanceId(), instance.getInstanceType(), ssiId, label); retVal.add(slave); } return retVal; } - private SpotinstSlave handleNewAwsInstance(String instanceId, String instanceType, String label) { + private SpotinstSlave handleNewAwsInstance(String instanceId, String instanceType, String ssiId, String label) { Integer executors = getNumOfExecutors(instanceType); addToPending(instanceId, executors, PendingInstance.StatusEnum.INSTANCE_INITIATING, label); - SpotinstSlave retVal = buildSpotinstSlave(instanceId, instanceType, String.valueOf(executors)); + SpotinstSlave retVal = buildSpotinstSlave(instanceId, instanceType, ssiId, String.valueOf(executors)); return retVal; } @@ -421,8 +427,21 @@ private Boolean isSlaveExistForInstance(AwsGroupInstance instance) { return retVal; } + private String getSsiByInstance(String instanceId) { + String retVal = null; + Optional optionalMatchingSsi = + ssiById.values().stream().filter(ssi -> instanceId.equals(ssi.getInstanceId())).findFirst(); + + if (optionalMatchingSsi.isPresent()) { + retVal = optionalMatchingSsi.get().getId(); + } + + return retVal; + } + private void addSpotinstSlave(AwsGroupInstance instance) { - SpotinstSlave slave = handleNewAwsInstance(instance.getInstanceId(), instance.getInstanceType(), null); + String ssiId = getSsiByInstance(instance.getInstanceId()); + SpotinstSlave slave = handleNewAwsInstance(instance.getInstanceId(), instance.getInstanceType(), ssiId, null); if (slave != null) { try { diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java index c769c526..1321eb4a 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java @@ -40,13 +40,13 @@ public class AzureSpotCloud extends BaseSpotinstCloud { @DataBoundConstructor public AzureSpotCloud(String groupId, String labelString, String idleTerminationMinutes, String workspaceDir, SlaveUsageEnum usage, String tunnel, Boolean shouldUseWebsocket, - Boolean shouldRetriggerBuilds, String vmargs, + SpotReTriggerBuilds spotReTriggerBuilds, String vmargs, EnvironmentVariablesNodeProperty environmentVariables, ToolLocationNodeProperty toolLocations, String accountId, ConnectionMethodEnum connectionMethod, ComputerConnector computerConnector, Boolean shouldUsePrivateIp, SpotGlobalExecutorOverride globalExecutorOverride, Integer pendingThreshold) { super(groupId, labelString, idleTerminationMinutes, workspaceDir, usage, tunnel, shouldUseWebsocket, - shouldRetriggerBuilds, vmargs, environmentVariables, toolLocations, accountId, connectionMethod, + spotReTriggerBuilds, vmargs, environmentVariables, toolLocations, accountId, connectionMethod, computerConnector, shouldUsePrivateIp, globalExecutorOverride, pendingThreshold); } //endregion @@ -97,8 +97,8 @@ public Boolean deallocateInstance(String instanceId) { @Override public Boolean detachInstance(String instanceId) { - boolean retVal = false; - IAzureVmGroupRepo azureVmGroupRepo = RepoManager.getInstance().getAzureVmGroupRepo(); + boolean retVal = false; + IAzureVmGroupRepo azureVmGroupRepo = RepoManager.getInstance().getAzureVmGroupRepo(); ApiResponse detachVmResponse = azureVmGroupRepo.detachVM(groupId, instanceId, this.accountId); if (detachVmResponse.isRequestSucceed()) { @@ -150,8 +150,8 @@ protected void internalSyncGroupInstances() { public Map getInstanceIpsById() { Map retVal = new HashMap<>(); - IAzureVmGroupRepo awsGroupRepo = RepoManager.getInstance().getAzureVmGroupRepo(); - ApiResponse> instancesResponse = awsGroupRepo.getGroupVms(groupId, accountId); + IAzureVmGroupRepo azureGroupRepo = RepoManager.getInstance().getAzureVmGroupRepo(); + ApiResponse> instancesResponse = azureGroupRepo.getGroupVms(groupId, accountId); if (instancesResponse.isRequestSucceed()) { List instances = instancesResponse.getValue(); @@ -216,7 +216,7 @@ private void removeOldSlaveInstances(List azureGroupVms) { if (allGroupsSlaves.size() > 0) { List elastigroupVmIds = - azureGroupVms.stream().filter(x -> x.getVmName() != null).map(AzureGroupVm::getVmName) + azureGroupVms.stream().map(AzureGroupVm::getVmName).filter(Objects::nonNull) .collect(Collectors.toList()); for (SpotinstSlave slave : allGroupsSlaves) { diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java index 0af037b4..7c61e5a5 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java @@ -38,14 +38,14 @@ public class AzureSpotinstCloud extends BaseSpotinstCloud { @DataBoundConstructor public AzureSpotinstCloud(String groupId, String labelString, String idleTerminationMinutes, String workspaceDir, SlaveUsageEnum usage, String tunnel, Boolean shouldUseWebsocket, - Boolean shouldRetriggerBuilds, String vmargs, + SpotReTriggerBuilds spotReTriggerBuilds, String vmargs, EnvironmentVariablesNodeProperty environmentVariables, ToolLocationNodeProperty toolLocations, String accountId, ConnectionMethodEnum connectionMethod, ComputerConnector computerConnector, Boolean shouldUsePrivateIp, SpotGlobalExecutorOverride globalExecutorOverride, Integer pendingThreshold) { super(groupId, labelString, idleTerminationMinutes, workspaceDir, usage, tunnel, shouldUseWebsocket, - shouldRetriggerBuilds, vmargs, environmentVariables, toolLocations, accountId, connectionMethod, + spotReTriggerBuilds, vmargs, environmentVariables, toolLocations, accountId, connectionMethod, computerConnector, shouldUsePrivateIp, globalExecutorOverride, pendingThreshold); } //endregion @@ -81,14 +81,14 @@ protected String getStatefulInstanceId(String instanceId) { } @Override - public Boolean deallocateInstance(String instanceId){ + public Boolean deallocateInstance(String instanceId) { return false;//TODO: implement } @Override public Boolean detachInstance(String instanceId) { - boolean retVal = false; - IAzureGroupRepo azureGroupRepo = RepoManager.getInstance().getAzureGroupRepo(); + boolean retVal = false; + IAzureGroupRepo azureGroupRepo = RepoManager.getInstance().getAzureGroupRepo(); ApiResponse detachInstanceResponse = azureGroupRepo.detachInstance(groupId, instanceId, accountId); if (detachInstanceResponse.isRequestSucceed()) { @@ -193,7 +193,7 @@ public Boolean onInstanceReady(String instanceId) { @Override protected PendingExecutorsCounts getPendingExecutors(ProvisionRequest request) { PendingExecutorsCounts retVal = new PendingExecutorsCounts(); - Integer pendingExecutors = 0; + int pendingExecutors = 0; Integer initiatingExecutors = 0; if (pendingInstances.size() > 0) { diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index 6e05903f..7522a2d6 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -14,7 +14,6 @@ import hudson.tools.ToolInstallation; import hudson.tools.ToolLocationNodeProperty; import jenkins.model.Jenkins; -import org.apache.commons.lang.BooleanUtils; import org.kohsuke.stapler.DataBoundSetter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,7 +40,7 @@ public abstract class BaseSpotinstCloud extends Cloud { protected String groupId; protected Map pendingInstances; protected Map slaveInstancesDetailsByInstanceId; - protected Map ssiByInstanceId; + protected Map ssiById; private String labelString; private String idleTerminationMinutes; private String workspaceDir; @@ -52,7 +51,7 @@ public abstract class BaseSpotinstCloud extends Cloud { private EnvironmentVariablesNodeProperty environmentVariables; private ToolLocationNodeProperty toolLocations; private Boolean shouldUseWebsocket; - private Boolean shouldRetriggerBuilds; + private SpotReTriggerBuilds spotReTriggerBuilds; private Boolean isSingleTaskNodesEnabled; private ComputerConnector computerConnector; private ConnectionMethodEnum connectionMethod; @@ -65,7 +64,7 @@ public abstract class BaseSpotinstCloud extends Cloud { //region Constructor public BaseSpotinstCloud(String groupId, String labelString, String idleTerminationMinutes, String workspaceDir, SlaveUsageEnum usage, String tunnel, Boolean shouldUseWebsocket, - Boolean shouldRetriggerBuilds, String vmargs, + SpotReTriggerBuilds spotReTriggerBuilds, String vmargs, EnvironmentVariablesNodeProperty environmentVariables, ToolLocationNodeProperty toolLocations, String accountId, ConnectionMethodEnum connectionMethod, ComputerConnector computerConnector, @@ -88,7 +87,13 @@ public BaseSpotinstCloud(String groupId, String labelString, String idleTerminat this.usage = SlaveUsageEnum.NORMAL; } - this.shouldRetriggerBuilds = shouldRetriggerBuilds == null || BooleanUtils.isTrue(shouldRetriggerBuilds); + if (spotReTriggerBuilds != null) { + this.spotReTriggerBuilds = spotReTriggerBuilds; + } + else { + this.spotReTriggerBuilds = new SpotReTriggerBuilds(false, false); + } + this.tunnel = tunnel; this.shouldUseWebsocket = shouldUseWebsocket; this.vmargs = vmargs; @@ -291,7 +296,7 @@ private void connectAgent(SpotinstSlave offlineAgent, String ipForAgent) { try { SpotSSHComputerLauncher launcher = new SpotSSHComputerLauncher(connector.launch(ipForAgent, computerForAgent.getListener()), - this.getShouldRetriggerBuilds()); + this.getShouldReTriggerBuilds()); offlineAgent.setLauncher(launcher); // save this information to disk - so Launcher survives Jenkins restarts offlineAgent.save(); @@ -495,7 +500,7 @@ protected void addToPending(String id, Integer numOfExecutors, PendingInstance.S pendingInstances.put(id, pendingInstance); } - protected SpotinstSlave buildSpotinstSlave(String id, String instanceType, String numOfExecutors) { + protected SpotinstSlave buildSpotinstSlave(String id, String instanceType, String ssiId, String numOfExecutors) { SpotinstSlave slave = null; Node.Mode mode = Node.Mode.NORMAL; @@ -507,8 +512,8 @@ protected SpotinstSlave buildSpotinstSlave(String id, String instanceType, Strin try { ComputerLauncher launcher = buildLauncherForAgent(id); - slave = new SpotinstSlave(id, groupId, id, instanceType, labelString, idleTerminationMinutes, workspaceDir, - numOfExecutors, mode, launcher, nodeProperties); + slave = new SpotinstSlave(id, groupId, id, instanceType, ssiId, labelString, idleTerminationMinutes, + workspaceDir, numOfExecutors, mode, launcher, nodeProperties); } catch (Descriptor.FormException | IOException e) { @@ -519,6 +524,10 @@ protected SpotinstSlave buildSpotinstSlave(String id, String instanceType, Strin return slave; } + protected SpotinstSlave buildSpotinstSlave(String id, String instanceType, String numOfExecutors) { + return buildSpotinstSlave(id, instanceType, null, numOfExecutors); + } + private ComputerLauncher buildLauncherForAgent(String instanceId) throws IOException { ComputerLauncher retVal; boolean isSshCloud = this.getConnectionMethod().equals(ConnectionMethodEnum.SSH); @@ -536,7 +545,7 @@ private ComputerLauncher buildLauncherForAgent(String instanceId) throws IOExcep private ComputerLauncher handleJNLPLauncher() { return new SpotinstComputerLauncher(this.getTunnel(), this.getVmargs(), this.getShouldUseWebsocket(), - this.getShouldRetriggerBuilds()); + this.getShouldReTriggerBuilds()); } private ComputerLauncher HandleSSHLauncher(SlaveInstanceDetails instanceDetailsById) throws IOException { @@ -554,7 +563,7 @@ private ComputerLauncher HandleSSHLauncher(SlaveInstanceDetails instanceDetailsB if (ipAddress != null) { try { - Boolean shouldRetriggerBuilds = this.getShouldRetriggerBuilds(); + Boolean shouldRetriggerBuilds = this.getShouldReTriggerBuilds(); retVal = new SpotSSHComputerLauncher( this.getComputerConnector().launch(instanceDetailsById.getPublicIp(), TaskListener.NULL), shouldRetriggerBuilds); @@ -743,12 +752,28 @@ public void setShouldUseWebsocket(Boolean shouldUseWebsocket) { this.shouldUseWebsocket = shouldUseWebsocket; } - public Boolean getShouldRetriggerBuilds() { - return shouldRetriggerBuilds; + public SpotReTriggerBuilds getSpotReTriggerBuilds() { + return spotReTriggerBuilds; } - public void setShouldRetriggerBuilds(Boolean shouldRetriggerBuilds) { - this.shouldRetriggerBuilds = shouldRetriggerBuilds; + public void setSpotReTriggerBuilds(SpotReTriggerBuilds spotReTriggerBuilds) { + this.spotReTriggerBuilds = spotReTriggerBuilds; + } + + public Boolean getShouldReTriggerBuilds() { + return getSpotReTriggerBuilds().getShouldReTriggerBuilds(); + } + + public void setShouldReTriggerBuilds(Boolean shouldReTriggerBuilds) { + this.getSpotReTriggerBuilds().setShouldReTriggerBuilds(shouldReTriggerBuilds); + } + + public Boolean getStickyNode() { + return getSpotReTriggerBuilds().getStickyNode(); + } + + public void setStickyNode(Boolean stickyNode) { + getSpotReTriggerBuilds().setStickyNode(stickyNode); } public ConnectionMethodEnum getConnectionMethod() { @@ -777,6 +802,7 @@ public boolean getShouldUsePrivateIp() { if (shouldUsePrivateIp == null) { return false; } + return shouldUsePrivateIp; } diff --git a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java index 6a7c0722..6d88ab65 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java @@ -42,14 +42,14 @@ public class GcpSpotinstCloud extends BaseSpotinstCloud { @DataBoundConstructor public GcpSpotinstCloud(String groupId, String labelString, String idleTerminationMinutes, String workspaceDir, SlaveUsageEnum usage, String tunnel, Boolean shouldUseWebsocket, - Boolean shouldRetriggerBuilds, String vmargs, + SpotReTriggerBuilds spotReTriggerBuilds, String vmargs, EnvironmentVariablesNodeProperty environmentVariables, ToolLocationNodeProperty toolLocations, String accountId, ConnectionMethodEnum connectionMethod, ComputerConnector computerConnector, Boolean shouldUsePrivateIp, SpotGlobalExecutorOverride globalExecutorOverride, Integer pendingThreshold) { super(groupId, labelString, idleTerminationMinutes, workspaceDir, usage, tunnel, shouldUseWebsocket, - shouldRetriggerBuilds, vmargs, environmentVariables, toolLocations, accountId, connectionMethod, + spotReTriggerBuilds, vmargs, environmentVariables, toolLocations, accountId, connectionMethod, computerConnector, shouldUsePrivateIp, globalExecutorOverride, pendingThreshold); } //endregion diff --git a/src/main/java/hudson/plugins/spotinst/cloud/SpotReTriggerBuilds.java b/src/main/java/hudson/plugins/spotinst/cloud/SpotReTriggerBuilds.java new file mode 100644 index 00000000..ac9cecfb --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/cloud/SpotReTriggerBuilds.java @@ -0,0 +1,68 @@ +package hudson.plugins.spotinst.cloud; + +import hudson.Extension; +import hudson.model.Describable; +import hudson.model.Descriptor; +import jenkins.model.Jenkins; +import org.apache.commons.lang.BooleanUtils; +import org.kohsuke.stapler.DataBoundConstructor; + +import javax.annotation.Nonnull; + +public class SpotReTriggerBuilds implements Describable { + //region members + private Boolean shouldReTriggerBuilds; + private Boolean stickyNode; + //endregion + + //region Ctor + @DataBoundConstructor + public SpotReTriggerBuilds(Boolean shouldReTriggerBuilds, Boolean stickyNode) { + this.shouldReTriggerBuilds = BooleanUtils.isTrue(shouldReTriggerBuilds); + this.stickyNode = this.shouldReTriggerBuilds && BooleanUtils.isTrue(stickyNode); + } + //endregion + + //region Overrides + @Override + public Descriptor getDescriptor() { + Descriptor retVal = Jenkins.get().getDescriptor(getClass()); + + if (retVal == null) { + throw new RuntimeException("Descriptor of type SpotReTriggerBuilds cannot be null"); + } + + return retVal; + } + //endregion + + //region Classes + @Extension + public static final class DescriptorImpl extends Descriptor { + + @Nonnull + @Override + public String getDisplayName() { + return "Spot ReTrigger Builds"; + } + } + //endregion + + //region getters & setters + public Boolean getShouldReTriggerBuilds() { + return shouldReTriggerBuilds; + } + + public void setShouldReTriggerBuilds(Boolean shouldReTriggerBuilds) { + this.shouldReTriggerBuilds = shouldReTriggerBuilds; + } + + public Boolean getStickyNode() { + return stickyNode; + } + + public void setStickyNode(Boolean stickyNode) { + this.stickyNode = stickyNode; + } + //endregio +} diff --git a/src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java b/src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java new file mode 100644 index 00000000..0fbf56ab --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java @@ -0,0 +1,65 @@ +package hudson.plugins.spotinst.queue; + +import hudson.Extension; +import hudson.model.Node; +import hudson.model.Queue; +import hudson.model.queue.CauseOfBlockage; +import hudson.model.queue.QueueTaskDispatcher; +import hudson.plugins.spotinst.slave.SpotinstSlave; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Extension +public class SpotQueueTaskDispatcher extends QueueTaskDispatcher { + //region members + private static final Logger LOGGER = LoggerFactory.getLogger(SpotQueueTaskDispatcher.class); + //endregion + + //region override methods + @Override + public CauseOfBlockage canTake(Node node, Queue.Task task) { + CauseOfBlockage retVal = null; + boolean isTaskReservedForStatefulSlave = task instanceof StatefulInterruptedTask; + + if (isTaskReservedForStatefulSlave) { + StatefulInterruptedTask statefulInterruptedTask = (StatefulInterruptedTask) task; + + if (node instanceof SpotinstSlave) { + SpotinstSlave slave = (SpotinstSlave) node; + String ssi = slave.getSsiId(); + boolean isStatefulSlave = StringUtils.isNotEmpty(ssi); + + if (isStatefulSlave) { + boolean canNodeTakeTask = ssi.equals(statefulInterruptedTask.getSsi()); + + if (canNodeTakeTask) { + LOGGER.info("node {} with ssi {} can take task {}", slave.getNodeName(), ssi, + statefulInterruptedTask); + } + else { + retVal = new InconsistentSsiCauseOfBlockage(); + } + } + else { + retVal = new InconsistentSsiCauseOfBlockage(); + } + } + else { + retVal = new InconsistentSsiCauseOfBlockage(); + } + } + + return retVal; + } + //endregion + + //region classes + private static class InconsistentSsiCauseOfBlockage extends CauseOfBlockage { + @Override + public String getShortDescription() { + return "task reserved for slave with specific ssi"; + } + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/queue/SsiByTaskMapper.java b/src/main/java/hudson/plugins/spotinst/queue/SsiByTaskMapper.java new file mode 100644 index 00000000..d448973e --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/queue/SsiByTaskMapper.java @@ -0,0 +1,31 @@ +package hudson.plugins.spotinst.queue; + +import hudson.model.Executor; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class SsiByTaskMapper { + //region members + private static final Map ssiByStatefulTask = new ConcurrentHashMap<>(); + //endregion + + //region methods + public static void putSsiByTask(String taskName, Executor executor, String ssiId){ + String key = generateKey(taskName, executor); + ssiByStatefulTask.put(key, ssiId); + } + + public static String removeSsiByTask(String taskName, Executor executor){ + String retVal; + String key = generateKey(taskName, executor); + retVal = ssiByStatefulTask.remove(key); + return retVal; + } + //endregion + + //region private methods + private static String generateKey(String taskName, Executor executor){ + return String.format("%s_%s", taskName, executor.getId()); + } +} diff --git a/src/main/java/hudson/plugins/spotinst/queue/StatefulInterruptedTask.java b/src/main/java/hudson/plugins/spotinst/queue/StatefulInterruptedTask.java new file mode 100644 index 00000000..bda03630 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/queue/StatefulInterruptedTask.java @@ -0,0 +1,74 @@ +package hudson.plugins.spotinst.queue; + +import hudson.model.Queue; + +import javax.annotation.CheckForNull; +import javax.annotation.Nonnull; +import java.io.IOException; + +public class StatefulInterruptedTask implements Queue.Task { + //region members + @Nonnull + private final String ssi; + @Nonnull + private final Queue.Task task; + //endregion + + //region ctor + public StatefulInterruptedTask(@Nonnull String ssi, @Nonnull Queue.Task task) { + this.ssi = ssi; + + if(task instanceof StatefulInterruptedTask){ + this.task = ((StatefulInterruptedTask) task).task; + } + else { + this.task = task; + } + } + //endregion + + //region overrides + @Override + public String getName() { + return task.getName(); + } + + @Override + public String getFullDisplayName() { + return task.getFullDisplayName(); + } + + @Override + public void checkAbortPermission() { + task.checkAbortPermission(); + } + + @Override + public boolean hasAbortPermission() { + return task.hasAbortPermission(); + } + + @Override + public String getUrl() { + return task.getUrl(); + } + + @Override + public String getDisplayName() { + return task.getDisplayName(); + } + + @CheckForNull + @Override + public Queue.Executable createExecutable() throws IOException { + return task.createExecutable(); + } + //endregion + + //region getters & setters + @Nonnull + public String getSsi() { + return ssi; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/queue/StatefulQueueDecisionHandler.java b/src/main/java/hudson/plugins/spotinst/queue/StatefulQueueDecisionHandler.java new file mode 100644 index 00000000..98b9b1f8 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/queue/StatefulQueueDecisionHandler.java @@ -0,0 +1,44 @@ +package hudson.plugins.spotinst.queue; + +import hudson.Extension; +import hudson.model.Action; +import hudson.model.Node; +import hudson.model.Queue; +import hudson.plugins.spotinst.slave.SpotinstSlave; +import jenkins.model.Jenkins; + +import java.util.List; +import java.util.Optional; + +@Extension +public class StatefulQueueDecisionHandler extends Queue.QueueDecisionHandler { + //region override method + @Override + public boolean shouldSchedule(Queue.Task task, List actions) { + boolean retVal = true; + + if (task instanceof StatefulInterruptedTask) { + StatefulInterruptedTask statefulInterruptedTask = (StatefulInterruptedTask) task; + String ssi = statefulInterruptedTask.getSsi(); + Jenkins jenkinsInstance = Jenkins.getInstanceOrNull(); + + if (jenkinsInstance != null) { + List nodes = jenkinsInstance.getNodes(); + Optional optionalSsi = + nodes.stream().filter(node -> node instanceof SpotinstSlave).map(node -> (SpotinstSlave) node) + .filter(spotinstSlave -> ssi.equals(spotinstSlave.getSsiId())).findFirst(); + + if (optionalSsi.isPresent()) { + SpotinstSlave matchedSlave = optionalSsi.get(); + retVal = matchedSlave.isAcceptingTasks(); + } + else { + retVal = false; + } + } + } + + return retVal; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java b/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java index c89c3dc2..5607b035 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java @@ -2,6 +2,8 @@ import hudson.model.*; import hudson.model.queue.SubTask; +import hudson.plugins.spotinst.queue.SsiByTaskMapper; +import hudson.plugins.spotinst.queue.StatefulInterruptedTask; import hudson.slaves.SlaveComputer; import org.apache.commons.lang.BooleanUtils; import org.slf4j.Logger; @@ -12,8 +14,7 @@ /** - * Created by Shibel Karmi Mansour on 31/12/2020. - * + * Created by Shibel Karmi Mansour on 31/12/2020.* * A helper class for handling common callbacks (like afterDisconnect) for different types of * ComputerLaunchers. */ @@ -23,7 +24,7 @@ class SpotLauncherHelper { //endregion //region Methods - static void handleDisconnect(final SlaveComputer computer, Boolean shouldRetriggerBuilds){ + static void handleDisconnect(final SlaveComputer computer, Boolean shouldRetriggerBuilds) { shouldRetriggerBuilds = resolveShouldRetriggerBuilds(shouldRetriggerBuilds); @@ -53,9 +54,20 @@ static void handleDisconnect(final SlaveComputer computer, Boolean shouldRetrigg actions = ((Actionable) executable).getActions(); } - LOGGER.info(String.format("RETRIGGERING: %s - WITH ACTIONS: %s", task, actions)); + String ssiByTaskName = SsiByTaskMapper.removeSsiByTask(task.getName(), executor); - Queue.getInstance().schedule2(task, 10, actions); + if (ssiByTaskName != null) { + StatefulInterruptedTask statefulInterruptedTask = + new StatefulInterruptedTask(ssiByTaskName, task); + LOGGER.info(String.format("RETRIGGERING Stateful Task: %s - WITH ACTIONS: %s", statefulInterruptedTask, actions)); + + Queue.getInstance().schedule2(statefulInterruptedTask, 10, actions); + } + else { + LOGGER.info(String.format("RETRIGGERING: %s - WITH ACTIONS: %s", task, actions)); + + Queue.getInstance().schedule2(task, 10, actions); + } } } diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java index 8c34a1b8..2902b7ad 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java @@ -4,7 +4,10 @@ import hudson.model.Node; import hudson.model.Queue; import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; +import hudson.plugins.spotinst.queue.SsiByTaskMapper; +import hudson.plugins.spotinst.queue.StatefulInterruptedTask; import hudson.slaves.SlaveComputer; +import org.apache.commons.lang.StringUtils; import org.kohsuke.stapler.HttpRedirect; import org.kohsuke.stapler.HttpResponse; import org.kohsuke.stapler.HttpResponses; @@ -41,8 +44,28 @@ public void taskAccepted(Executor executor, Queue.Task task) { LOGGER.info(msg); this.setAcceptingTasks(false); SpotinstNonLocalizable spotinstNonLocalizable = new SpotinstNonLocalizable(msg); - SpotinstSingleTaskOfflineCause spotinstSingleTaskOfflineCause = new SpotinstSingleTaskOfflineCause(spotinstNonLocalizable); - this.setTemporarilyOffline(true,spotinstSingleTaskOfflineCause); + SpotinstSingleTaskOfflineCause spotinstSingleTaskOfflineCause = + new SpotinstSingleTaskOfflineCause(spotinstNonLocalizable); + this.setTemporarilyOffline(true, spotinstSingleTaskOfflineCause); + } + + if(spotinstCloud.getStickyNode()) { + String spotinstNodeSsiId = spotinstNode.getSsiId(); + + if (StringUtils.isNotEmpty(spotinstNodeSsiId)) { + SsiByTaskMapper.putSsiByTask(task.getName(), executor, spotinstNodeSsiId); + + //TODO: remove - only for logs + if (task instanceof StatefulInterruptedTask) { + StatefulInterruptedTask statefulInterruptedTask = (StatefulInterruptedTask) task; + String statefulTaskSsiId = statefulInterruptedTask.getSsi(); + + if (spotinstNodeSsiId.equals(statefulTaskSsiId) == false) { + LOGGER.warn("stateful task is reserved for ssi {}, however it is running on ssi {}", + statefulTaskSsiId, spotinstNodeSsiId); + } + } + } } } else { @@ -50,12 +73,19 @@ public void taskAccepted(Executor executor, Queue.Task task) { "Node %s has accepted a job but can't determine 'Single Task Nodes' setting because SpotinstNode's SpotinstCloud appears to be null.", spotinstNode.getNodeName())); } - } else { + } + else { LOGGER.error(String.format( - "Executor of Node %s has accepted a job but can't determine 'Single Task Nodes' setting because SpotinstNode is null.", executor.getOwner().getName())); + "Executor of Node %s has accepted a job but can't determine 'Single Task Nodes' setting because SpotinstNode is null.", + executor.getOwner().getName())); } } + @Override + public void taskCompleted(Executor executor, Queue.Task task, long durationMS){ + super.taskCompleted(executor, task, durationMS); + SsiByTaskMapper.removeSsiByTask(task.getName(), executor); + } //endregion //region Constructor @@ -94,7 +124,8 @@ public HttpResponse doDoDelete() throws IOException { } return new HttpRedirect(".."); - } catch (NullPointerException ex) { + } + catch (NullPointerException ex) { return HttpResponses.error(500, ex); } } @@ -114,6 +145,5 @@ public void resyncNode() { this.setNode(node); } } - //endregion } diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java index 6e71c91d..4f0c9c20 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java @@ -25,6 +25,7 @@ public class SpotinstSlave extends Slave implements EphemeralNode { private String instanceId; private String instanceType; + private String ssiId; private String elastigroupId; private String workspaceDir; private String groupUrl; @@ -34,9 +35,9 @@ public class SpotinstSlave extends Slave implements EphemeralNode { //endregion //region Constructor - public SpotinstSlave(String name, String elastigroupId, String instanceId, String instanceType, String label, - String idleTerminationMinutes, String workspaceDir, String numOfExecutors, Mode mode, - ComputerLauncher launcher, + public SpotinstSlave(String name, String elastigroupId, String instanceId, String instanceType, String ssiId, + String label, String idleTerminationMinutes, String workspaceDir, String numOfExecutors, + Mode mode, ComputerLauncher launcher, List> nodeProperties) throws Descriptor.FormException, IOException { @@ -45,6 +46,7 @@ public SpotinstSlave(String name, String elastigroupId, String instanceId, Strin this.elastigroupId = elastigroupId; this.instanceType = instanceType; + this.ssiId = ssiId; this.instanceId = instanceId; this.workspaceDir = workspaceDir; this.usage = SlaveUsageEnum.fromMode(mode); @@ -66,6 +68,14 @@ public String getInstanceType() { return instanceType; } + public String getSsiId() { + return ssiId; + } + + public void setSsiId(String ssiId) { + this.ssiId = ssiId; + } + public String getWorkspaceDir() { return workspaceDir; } diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/BaseSpotinstCloud/config.jelly b/src/main/resources/hudson/plugins/spotinst/cloud/BaseSpotinstCloud/config.jelly index 6f39a422..bf795e42 100644 --- a/src/main/resources/hudson/plugins/spotinst/cloud/BaseSpotinstCloud/config.jelly +++ b/src/main/resources/hudson/plugins/spotinst/cloud/BaseSpotinstCloud/config.jelly @@ -1,5 +1,5 @@ - + @@ -25,7 +25,7 @@ - + ${it.name()} @@ -64,14 +64,8 @@ - - - - - - - - + + @@ -129,7 +123,7 @@ Enabling this setting effectively means: override the Default executor count setting and ignore any instance-weight overrides if those are set. - + diff --git a/src/main/resources/hudson/plugins/spotinst/cloud/SpotReTriggerBuilds/config.jelly b/src/main/resources/hudson/plugins/spotinst/cloud/SpotReTriggerBuilds/config.jelly new file mode 100644 index 00000000..d4d2e056 --- /dev/null +++ b/src/main/resources/hudson/plugins/spotinst/cloud/SpotReTriggerBuilds/config.jelly @@ -0,0 +1,10 @@ + + + + + + Restrict re-triggered builds on the original ssi + In case of a stateful group. The rebuilds will use recycled resources according to stateful configuration. + + + \ No newline at end of file diff --git a/src/test/java/hudson/plugins/spotinst/SpotinstCloudTest.java b/src/test/java/hudson/plugins/spotinst/SpotinstCloudTest.java index 621b8521..5f7b22e4 100644 --- a/src/test/java/hudson/plugins/spotinst/SpotinstCloudTest.java +++ b/src/test/java/hudson/plugins/spotinst/SpotinstCloudTest.java @@ -72,12 +72,12 @@ private void setUpLockRepo() { RepoManager.getInstance().setLockRepo(lockRepo); } - private PendingInstance buildPendingInstance(String id, PendingInstance.StatusEnum status, Integer executors) { + private PendingInstance buildPendingInstance(String id, Integer executors) { PendingInstance pendingInstance = new PendingInstance(); pendingInstance.setCreatedAt(new Date()); pendingInstance.setNumOfExecutors(executors); pendingInstance.setId(id); - pendingInstance.setStatus(status); + pendingInstance.setStatus(PendingInstance.StatusEnum.PENDING); return pendingInstance; } @@ -87,10 +87,10 @@ private PendingInstance buildPendingInstance(String id, PendingInstance.StatusEn public void testAwsProvision_whenThereArePendingInsatcnesForAllExecutors_thenShouldNotSacleUp() { String groupId = "sig-1"; BaseSpotinstCloud spotinstCloud = - new AwsSpotinstCloud(groupId, "", "20", "/tmp", null, SlaveUsageEnum.NORMAL, "", false, true, "", null, + new AwsSpotinstCloud(groupId, "", "20", "/tmp", null, SlaveUsageEnum.NORMAL, "", false, new SpotReTriggerBuilds(true, false), "", null, null, null, null, null, null, null, null); Map pendingInstances = new HashMap<>(); - pendingInstances.put("sir-1", buildPendingInstance("sir-1", PendingInstance.StatusEnum.PENDING, 2)); + pendingInstances.put("sir-1", buildPendingInstance("sir-1", 2)); spotinstCloud.setPendingInstances(pendingInstances); spotinstCloud.provision(null, 2); Mockito.verify(RepoManager.getInstance().getAwsGroupRepo(), Mockito.never()) @@ -102,12 +102,12 @@ public void testAwsProvision_whenThereArePendingInsatcnesForPartOfTheExecutors_t String groupId = "sig-1"; String accountId = "act-111"; AwsSpotinstCloud spotinstCloud = - new AwsSpotinstCloud(groupId, "", "20", "/tmp", null, SlaveUsageEnum.NORMAL, "", false, true, "", null, + new AwsSpotinstCloud(groupId, "", "20", "/tmp", null, SlaveUsageEnum.NORMAL, "", false, new SpotReTriggerBuilds(true, false), "", null, null, accountId, null, null, null, null, null); jenkinsRule.jenkins.clouds.add(spotinstCloud); Map pendingInstances = new HashMap<>(); - pendingInstances.put("sir-1", buildPendingInstance("sir-1", PendingInstance.StatusEnum.PENDING, 2)); + pendingInstances.put("sir-1", buildPendingInstance("sir-1", 2)); spotinstCloud.setPendingInstances(pendingInstances); AwsScaleUpResult result = new AwsScaleUpResult(); AwsScaleResultNewSpot newSpot = new AwsScaleResultNewSpot(); @@ -160,6 +160,7 @@ public void testAwCloud_whenNoConnectionMethodIsProvided_thenDefaultIsJNLP() { for (Node node : allNodes) { SpotinstComputer computer = (SpotinstComputer) node.toComputer(); + assert computer != null; assertEquals(computer.getLauncher().getClass(), SpotinstComputerLauncher.class); } @@ -207,6 +208,7 @@ public void testAwsCloud_whenSshConnectionMethod_andIpIsAvailable_thenCreateSshL spotCloud.monitorInstances(); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(incomingInstance.getInstanceId()); + assert agent != null; assertEquals(agent.getLauncher().getClass(), SpotSSHComputerLauncher.class); } @@ -255,6 +257,7 @@ public void testAwsCloud_whenSshConnectionMethod_andIpIsNotAvailable_thenDoNotCo SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode("i-2"); + assert agent != null; assertNotEquals(agent.getLauncher().getClass(), SpotSSHComputerLauncher.class); // but neither create our JNLP launcher assertNotEquals(agent.getLauncher().getClass(), SpotinstComputerLauncher.class); @@ -268,7 +271,7 @@ public void testAwCloud_whenUsePrivateIpIsNull_thenUsePublicIp() { new AwsSpotinstCloud(groupId, "", "20", "/tmp", null, null, "", true, null, null, null, null, null, null, null, null, null, null); - assertEquals(spotCloud.getShouldUsePrivateIp(), false); + assertFalse(spotCloud.getShouldUsePrivateIp()); } @Test @@ -278,7 +281,7 @@ public void testAwCloud_whenUsePrivateIpIsTrue_thenUsePrivateIp() { new AwsSpotinstCloud(groupId, "", "20", "/tmp", null, null, "", true, null, null, null, null, null, null, null, true, null, null); - assertEquals(spotCloud.getShouldUsePrivateIp(), true); + assertTrue(spotCloud.getShouldUsePrivateIp()); } @Test @@ -288,7 +291,7 @@ public void testAwCloud_whenUsePrivateIpIsFalse_thenUsePublicIp() { new AwsSpotinstCloud(groupId, "", "20", "/tmp", null, null, "", true, null, null, null, null, null, null, null, false, null, null); - assertEquals(spotCloud.getShouldUsePrivateIp(), false); + assertFalse(spotCloud.getShouldUsePrivateIp()); } //endregion @@ -297,10 +300,10 @@ public void testAwCloud_whenUsePrivateIpIsFalse_thenUsePublicIp() { public void testGcpProvision_whenThereArePendingInsatcnesForAllExecutors_thenShouldNotSacleUp() { String groupId = "sig-1"; BaseSpotinstCloud spotinstCloud = - new GcpSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, true, "", null, null, null, null, null, + new GcpSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, null, null, null, null, null, null); Map pendingInstances = new HashMap<>(); - pendingInstances.put("sin-1", buildPendingInstance("sin-1", PendingInstance.StatusEnum.PENDING, 2)); + pendingInstances.put("sin-1", buildPendingInstance("sin-1", 2)); spotinstCloud.setPendingInstances(pendingInstances); spotinstCloud.provision(null, 2); Mockito.verify(RepoManager.getInstance().getGcpGroupRepo(), Mockito.never()) @@ -312,11 +315,11 @@ public void testGcpProvision_whenThereArePendingInsatcnesForPartOfTheExecutors_t String groupId = "sig-1"; String accountId = "act-111"; GcpSpotinstCloud spotinstCloud = - new GcpSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, true, "", null, null, accountId, null, + new GcpSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, accountId, null, null, null, null, null); jenkinsRule.jenkins.clouds.add(spotinstCloud); Map pendingInstances = new HashMap<>(); - pendingInstances.put("sin-1", buildPendingInstance("sin-1", PendingInstance.StatusEnum.PENDING, 2)); + pendingInstances.put("sin-1", buildPendingInstance("sin-1", 2)); spotinstCloud.setPendingInstances(pendingInstances); GcpScaleUpResult result = new GcpScaleUpResult(); @@ -346,11 +349,11 @@ public void testGcpProvision_whenThereArePendingInsatcnesForPartOfTheExecutors_t public void testAzureProvision_whenThereArePendingInsatcnesForAllExecutors_thenShouldNotSacleUp() { String groupId = "sig-1"; BaseSpotinstCloud spotinstCloud = - new AzureSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, false, "", null, null, null, null, + new AzureSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(false, false), "", null, null, null, null, null, null, null, null); Map pendingInstances = new HashMap<>(); - pendingInstances.put("q3213", buildPendingInstance(groupId, PendingInstance.StatusEnum.PENDING, 1)); - pendingInstances.put("41234", buildPendingInstance(groupId, PendingInstance.StatusEnum.PENDING, 1)); + pendingInstances.put("q3213", buildPendingInstance(groupId, 1)); + pendingInstances.put("41234", buildPendingInstance(groupId, 1)); spotinstCloud.setPendingInstances(pendingInstances); spotinstCloud.provision(null, 2); Mockito.verify(RepoManager.getInstance().getAzureGroupRepo(), Mockito.never()) @@ -362,16 +365,16 @@ public void testAzureProvision_whenThereArePendingInsatcnesForPartOfTheExecutors String groupId = "sig-1"; String accountId = "act-111"; AzureSpotinstCloud spotinstCloud = - new AzureSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, false, "", null, null, accountId, + new AzureSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(false, false), "", null, null, accountId, null, null, null, null, null); Map pendingInstances = new HashMap<>(); - pendingInstances.put("asda", buildPendingInstance(groupId, PendingInstance.StatusEnum.PENDING, 1)); - pendingInstances.put("ada", buildPendingInstance(groupId, PendingInstance.StatusEnum.PENDING, 1)); + pendingInstances.put("asda", buildPendingInstance(groupId, 1)); + pendingInstances.put("ada", buildPendingInstance(groupId, 1)); spotinstCloud.setPendingInstances(pendingInstances); Mockito.when(RepoManager.getInstance().getAzureGroupRepo() .scaleUp(Mockito.anyString(), Mockito.anyInt(), Mockito.anyString())) - .thenReturn(new ApiResponse(new Boolean(true))); + .thenReturn(new ApiResponse<>(Boolean.TRUE)); spotinstCloud.provision(null, 4); @@ -390,11 +393,11 @@ public void testAzureProvision_whenThereArePendingInsatcnesForPartOfTheExecutors public void testAzureV3Provision_whenThereArePendingInstancesForAllExecutors_thenShouldNotScaleUp() { String groupId = "sig-1"; BaseSpotinstCloud spotinstCloud = - new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, true, "", null, null, null, null, null, + new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, null, null, null, null, null, null); Map pendingInstances = new HashMap<>(); - pendingInstances.put("vm-1", buildPendingInstance("vm-1", PendingInstance.StatusEnum.PENDING, 2)); + pendingInstances.put("vm-1", buildPendingInstance("vm-1", 2)); spotinstCloud.setPendingInstances(pendingInstances); spotinstCloud.provision(null, 2); @@ -407,11 +410,11 @@ public void testAzureV3Provision_whenThereArePendingInstancesForPartOfTheExecuto String groupId = "sig-1"; String accountId = "act-111"; BaseSpotinstCloud spotinstCloud = - new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, true, "", null, null, accountId, null, + new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, accountId, null, null, null, null, null); jenkinsRule.jenkins.clouds.add(spotinstCloud); Map pendingInstances = new HashMap<>(); - pendingInstances.put("vm-1", buildPendingInstance("vm-1", PendingInstance.StatusEnum.PENDING, 2)); + pendingInstances.put("vm-1", buildPendingInstance("vm-1", 2)); spotinstCloud.setPendingInstances(pendingInstances); AzureScaleUpResultNewVm newSpot = new AzureScaleUpResultNewVm(); @@ -442,7 +445,7 @@ public void testAzureV3Provision_whenUnrecognizedVmSize_thenDefaultTo1Executor() String groupId = "sig-1"; String accountId = "act-111"; BaseSpotinstCloud spotinstCloud = - new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, true, "", null, null, accountId, null, + new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, accountId, null, null, null, null, null); jenkinsRule.jenkins.clouds.add(spotinstCloud); AzureScaleUpResultNewVm newSpot = new AzureScaleUpResultNewVm(); @@ -460,6 +463,7 @@ public void testAzureV3Provision_whenUnrecognizedVmSize_thenDefaultTo1Executor() Node node = Jenkins.get().getNode("vm-2"); assertNotEquals(node, null); + assert node != null; assertEquals(1, node.getNumExecutors()); } @@ -468,7 +472,7 @@ public void testAzureV3Provision_whenNewInstancesAreLaunched_thenTheirSizeIsAcco String groupId = "sig-1"; String accountId = "act-111"; BaseSpotinstCloud spotinstCloud = - new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, true, "", null, null, accountId, null, + new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, accountId, null, null, null, null, null); jenkinsRule.jenkins.clouds.add(spotinstCloud); AzureVmSizeEnum vmSizeBasicA1 = AzureVmSizeEnum.BASIC_A1; @@ -505,11 +509,11 @@ public void testAzureV3Provision_whenNewInstancesAreLaunched_thenTheirSizeIsAcco String groupId = "sig-1"; String accountId = "act-111"; BaseSpotinstCloud spotinstCloud = - new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, true, "", null, null, accountId, null, + new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, accountId, null, null, null, null, null); jenkinsRule.jenkins.clouds.add(spotinstCloud); Map pendingInstances = new HashMap<>(); - pendingInstances.put("vm-1", buildPendingInstance("vm-1", PendingInstance.StatusEnum.PENDING, 2)); + pendingInstances.put("vm-1", buildPendingInstance("vm-1", 2)); spotinstCloud.setPendingInstances(pendingInstances); AzureVmSizeEnum vmSizeBasicA1 = AzureVmSizeEnum.BASIC_A1; @@ -544,7 +548,7 @@ public void testAzureV3Provision_whenNewInstancesAreLaunched_thenTheirSizeIsAcco public void testAzureV3Cloud_whenNoConnectionMethodIsProvided_thenDefaultIsJNLP() { String groupId = "sig-1"; BaseSpotinstCloud spotCloud = - new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, true, "", null, null, null, null, null, + new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, null, null, null, null, null, null); jenkinsRule.jenkins.clouds.add(spotCloud); @@ -568,6 +572,7 @@ public void testAzureV3Cloud_whenNoConnectionMethodIsProvided_thenDefaultIsJNLP( for (Node node : allNodes) { SpotinstComputer computer = (SpotinstComputer) node.toComputer(); + assert computer != null; assertEquals(computer.getLauncher().getClass(), SpotinstComputerLauncher.class); } @@ -627,6 +632,7 @@ public void testGlobalExecutorOverride_whenIsEnabledAndInstanceTypeIsMatchedInEn cloud.provision(null, 1); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); + assert agent != null; assertEquals(agent.getNumExecutors(), globalOverride.getExecutors().intValue()); } @@ -667,6 +673,7 @@ public void testGlobalExecutorOverride_whenIsEnabledAndInstanceTypeIsNotMatchedI cloud.provision(null, 1); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); + assert agent != null; assertEquals(agent.getNumExecutors(), globalOverride.getExecutors().intValue()); } @@ -713,6 +720,7 @@ null, accountId, ConnectionMethodEnum.SSH, getSSHConnector(), false, cloud.provision(null, 1); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); + assert agent != null; assertEquals(agent.getNumExecutors(), expectedExecutors); } @@ -755,6 +763,7 @@ public void testGlobalExecutorOverride_whenIsDisabledAndInstanceTypeIsNotMatched cloud.provision(null, 1); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); + assert agent != null; assertEquals(agent.getNumExecutors(), 1); } @@ -795,6 +804,7 @@ public void testGlobalExecutorOverride_whenIsDisabledAndInstanceTypeIsMatchedInE cloud.provision(null, 1); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); + assert agent != null; assertEquals(agent.getNumExecutors(), AwsInstanceTypeEnum.C4Large.getExecutors().intValue()); } @@ -842,6 +852,7 @@ null, accountId, ConnectionMethodEnum.SSH, getSSHConnector(), false, cloud.provision(null, 1); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); + assert agent != null; assertEquals(agent.getNumExecutors(), expectedExecutors); } @@ -883,6 +894,7 @@ public void testGlobalExecutorOverride_whenIsInvalidNegativeAndInstanceTypeIsMat cloud.provision(null, 1); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); + assert agent != null; assertEquals(agent.getNumExecutors(), AwsInstanceTypeEnum.C4Large.getExecutors().intValue()); } @@ -923,6 +935,7 @@ public void testGlobalExecutorOverride_whenIsInvalidNegativeAndInstanceTypeIsNot cloud.provision(null, 1); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); + assert agent != null; assertEquals(agent.getNumExecutors(), 1); } @@ -970,6 +983,7 @@ null, accountId, ConnectionMethodEnum.SSH, getSSHConnector(), false, cloud.provision(null, 1); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); + assert agent != null; assertEquals(agent.getNumExecutors(), expectedExecutors); } @@ -1010,6 +1024,7 @@ public void testGlobalExecutorOverride_whenIsInvalidNullAndInstanceTypeIsMatched cloud.provision(null, 1); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); + assert agent != null; assertEquals(agent.getNumExecutors(), AwsInstanceTypeEnum.C4Large.getExecutors().intValue()); } @@ -1050,6 +1065,7 @@ public void testGlobalExecutorOverride_whenIsInvalidNullAndInstanceTypeIsNotMatc cloud.provision(null, 1); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); + assert agent != null; assertEquals(agent.getNumExecutors(), 1); } @@ -1097,6 +1113,7 @@ null, accountId, ConnectionMethodEnum.SSH, getSSHConnector(), false, cloud.provision(null, 1); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); + assert agent != null; assertEquals(agent.getNumExecutors(), expectedExecutors); } @@ -1138,6 +1155,7 @@ public void testGlobalExecutorOverride_whenIsNullAndInstanceTypeIsMatchedInEnum_ cloud.provision(null, 1); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); + assert agent != null; assertEquals(agent.getNumExecutors(), AwsInstanceTypeEnum.C4Large.getExecutors().intValue()); } @@ -1177,6 +1195,7 @@ public void testGlobalExecutorOverride_whenIsNullAndInstanceTypeIsNotMatchedInEn cloud.provision(null, 1); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); + assert agent != null; assertEquals(agent.getNumExecutors(), 1); } @@ -1224,6 +1243,7 @@ null, accountId, ConnectionMethodEnum.SSH, getSSHConnector(), false, cloud.provision(null, 1); SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); + assert agent != null; assertEquals(agent.getNumExecutors(), expectedExecutors); } @@ -1275,6 +1295,7 @@ public void testSpotinstSlaveTermination_ifAgentInPendingInstances_thenAgentIsRe SpotinstSlave agent = (SpotinstSlave) Jenkins.get().getNode(newSpot.getInstanceId()); assertEquals(cloud.isInstancePending(newSpot.getInstanceId()), true); + assert agent != null; agent.terminate(); assertEquals(cloud.isInstancePending(newSpot.getInstanceId()), false); @@ -1291,7 +1312,7 @@ public void testAzureSpotinstCloud_DescriptorReturnsAzureSpotinstCloudString() { String groupId = "sig-1"; String accountId = "act-111"; BaseSpotinstCloud spotinstCloud = - new AzureSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, false, "", null, null, accountId, + new AzureSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(false, false), "", null, null, accountId, null, null, null, null, null); assertTrue(spotinstCloud.getDescriptor().toString().contains("AzureSpotinstCloud")); From 58de2625f109029b653b122ffc8e5406ab59507e Mon Sep 17 00:00:00 2001 From: sitay Date: Thu, 31 Aug 2023 23:22:04 +0300 Subject: [PATCH 06/13] initial commit --- .../spotinst/cloud/AwsSpotinstCloud.java | 20 +++-- .../spotinst/cloud/BaseSpotinstCloud.java | 2 - .../common/StatefulInstanceStateEnum.java | 62 ++++++++++++++ .../model/aws/AwsStatefulInstance.java | 14 ++- .../aws/AwsStatefulInstancesManager.java | 39 +++++++++ .../queue/SpotQueueTaskDispatcher.java | 85 +++++++++++++++---- .../queue/StatefulQueueDecisionHandler.java | 63 +++++++++++--- .../plugins/spotinst/SpotinstCloudTest.java | 4 +- 8 files changed, 247 insertions(+), 42 deletions(-) create mode 100644 src/main/java/hudson/plugins/spotinst/common/StatefulInstanceStateEnum.java create mode 100644 src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java index 168aa221..d0f8028e 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java @@ -41,8 +41,8 @@ public class AwsSpotinstCloud extends BaseSpotinstCloud { @DataBoundConstructor public AwsSpotinstCloud(String groupId, String labelString, String idleTerminationMinutes, String workspaceDir, List executorsForTypes, SlaveUsageEnum usage, - String tunnel, Boolean shouldUseWebsocket, SpotReTriggerBuilds spotReTriggerBuilds, String vmargs, - EnvironmentVariablesNodeProperty environmentVariables, + String tunnel, Boolean shouldUseWebsocket, SpotReTriggerBuilds spotReTriggerBuilds, + String vmargs, EnvironmentVariablesNodeProperty environmentVariables, ToolLocationNodeProperty toolLocations, String accountId, ConnectionMethodEnum connectionMethod, ComputerConnector computerConnector, Boolean shouldUsePrivateIp, SpotGlobalExecutorOverride globalExecutorOverride, @@ -186,7 +186,6 @@ protected void internalSyncGroupInstances() { this.slaveInstancesDetailsByInstanceId = new HashMap<>(slaveInstancesDetailsByInstanceId); syncGroupStatefulInstances(); - addNewSlaveInstances(instances); removeOldSlaveInstances(instances); } @@ -280,7 +279,7 @@ private void syncGroupStatefulInstances() { Map ssiById = new HashMap<>(); statefulInstances.forEach(ssi -> ssiById.put(ssi.getId(), ssi)); - this.ssiById = ssiById; + AwsStatefulInstancesManager.getAwsStatefulInstanceBySsiById().put(groupId, ssiById); } private List handleNewAwsSpots(AwsScaleUpResult scaleUpResult, String label) { @@ -429,11 +428,16 @@ private Boolean isSlaveExistForInstance(AwsGroupInstance instance) { private String getSsiByInstance(String instanceId) { String retVal = null; - Optional optionalMatchingSsi = - ssiById.values().stream().filter(ssi -> instanceId.equals(ssi.getInstanceId())).findFirst(); + Map ssiById = + AwsStatefulInstancesManager.getAwsStatefulInstanceBySsiById().get(groupId); + + if(ssiById != null) { + Optional optionalMatchingSsi = + ssiById.values().stream().filter(ssi -> instanceId.equals(ssi.getInstanceId())).findFirst(); - if (optionalMatchingSsi.isPresent()) { - retVal = optionalMatchingSsi.get().getId(); + if (optionalMatchingSsi.isPresent()) { + retVal = optionalMatchingSsi.get().getId(); + } } return retVal; diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index 7522a2d6..b910c0fb 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -5,7 +5,6 @@ import hudson.model.labels.LabelAtom; import hudson.plugins.spotinst.api.infra.JsonMapper; import hudson.plugins.spotinst.common.*; -import hudson.plugins.spotinst.model.aws.AwsStatefulInstance; import hudson.plugins.spotinst.slave.*; import hudson.plugins.sshslaves.SSHConnector; import hudson.slaves.*; @@ -40,7 +39,6 @@ public abstract class BaseSpotinstCloud extends Cloud { protected String groupId; protected Map pendingInstances; protected Map slaveInstancesDetailsByInstanceId; - protected Map ssiById; private String labelString; private String idleTerminationMinutes; private String workspaceDir; diff --git a/src/main/java/hudson/plugins/spotinst/common/StatefulInstanceStateEnum.java b/src/main/java/hudson/plugins/spotinst/common/StatefulInstanceStateEnum.java new file mode 100644 index 00000000..6ac57229 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/common/StatefulInstanceStateEnum.java @@ -0,0 +1,62 @@ +package hudson.plugins.spotinst.common; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Created by sitay on 30/08/23. + */ +public enum StatefulInstanceStateEnum { + //region Enums + ACTIVE("ACTIVE"), + PAUSE("PAUSE"), + PAUSING("PAUSING"), + PAUSED("PAUSED"), + RESUME("RESUME"), + RESUMING("RESUMING"), + RECYCLE("RECYCLE"), + RECYCLING("RECYCLING"), + DEALLOCATE("DEALLOCATE"), + DEALLOCATING("DEALLOCATING"), + DEALLOCATED("DEALLOCATED"), + ERROR("ERROR"); + //endregion + + //region Members + private static final Logger LOGGER = LoggerFactory.getLogger(StatefulInstanceStateEnum.class); + private final String name; + //endregion + + //region Constructors + StatefulInstanceStateEnum(String name) { + this.name = name; + } + //endregion + + //region Getters & Setters + public String getName() { + return name; + } + //endregion + + //region Methods + public static StatefulInstanceStateEnum fromName(String name) { + StatefulInstanceStateEnum retVal = null; + + for (StatefulInstanceStateEnum deploymentInstanceStatus : StatefulInstanceStateEnum.values()) { + if (name.equalsIgnoreCase(deploymentInstanceStatus.name)) { + retVal = deploymentInstanceStatus; + break; + } + } + + if (retVal == null) { + LOGGER.error( + "Tried to create stateful instance state Enum for: " + name + ", but we don't support such type "); + } + + return retVal; + } + //endregion +} + diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstance.java b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstance.java index f5ef9521..8836e08f 100644 --- a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstance.java +++ b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstance.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; +import hudson.plugins.spotinst.common.StatefulInstanceStateEnum; /** * Created by ItayShklar on 07/08/2023. @@ -10,8 +11,9 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class AwsStatefulInstance { //region members - private String id; - private String instanceId; + private String id; + private String instanceId; + private StatefulInstanceStateEnum state; //endregion //region getters & setters @@ -30,5 +32,13 @@ public String getInstanceId() { public void setInstanceId(String instanceId) { this.instanceId = instanceId; } + + public StatefulInstanceStateEnum getState() { + return state; + } + + public void setState(StatefulInstanceStateEnum state) { + this.state = state; + } //endregion } diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java new file mode 100644 index 00000000..afe8822d --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java @@ -0,0 +1,39 @@ +package hudson.plugins.spotinst.model.aws; + +import java.util.Collection; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Created by sitay on 30/08/2013. + */ +public class AwsStatefulInstancesManager { + //region members + private final static Map> awsStatefulInstanceBySsiById = + new ConcurrentHashMap<>(); + //endregion + + //region methods + public static AwsStatefulInstance getStatefulInstanceBySSi(String ssi){ + AwsStatefulInstance retVal = null; + Collection> allGroupsSsiById = + awsStatefulInstanceBySsiById.values(); + + Optional optionalMatchingStatefulInstance = + allGroupsSsiById.stream().filter(groupSsiById -> groupSsiById.containsKey(ssi)) + .map(groupSsiById -> groupSsiById.get(ssi)).findFirst(); + + if (optionalMatchingStatefulInstance.isPresent()) { + retVal = optionalMatchingStatefulInstance.get(); + } + + return retVal; + } + + //region getters & setters + public static Map> getAwsStatefulInstanceBySsiById() { + return awsStatefulInstanceBySsiById; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java b/src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java index 0fbf56ab..5d62ff34 100644 --- a/src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java +++ b/src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java @@ -5,11 +5,17 @@ import hudson.model.Queue; import hudson.model.queue.CauseOfBlockage; import hudson.model.queue.QueueTaskDispatcher; +import hudson.plugins.spotinst.model.aws.AwsStatefulInstance; +import hudson.plugins.spotinst.model.aws.AwsStatefulInstancesManager; import hudson.plugins.spotinst.slave.SpotinstSlave; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +/** + * Created by sitay on 30/08/2013. + */ @Extension public class SpotQueueTaskDispatcher extends QueueTaskDispatcher { //region members @@ -24,22 +30,58 @@ public CauseOfBlockage canTake(Node node, Queue.Task task) { if (isTaskReservedForStatefulSlave) { StatefulInterruptedTask statefulInterruptedTask = (StatefulInterruptedTask) task; + String statefulTaskSsi = statefulInterruptedTask.getSsi(); + AwsStatefulInstance matchingStatefulInstance = + AwsStatefulInstancesManager.getStatefulInstanceBySSi(statefulTaskSsi); + boolean isSsiExist = matchingStatefulInstance != null; + + if (isSsiExist) { + switch (matchingStatefulInstance.getState()) { + case PAUSE: + case PAUSING: + case PAUSED: + case DEALLOCATE: + case DEALLOCATING: + case DEALLOCATED: + case ERROR: + //SSI isn't activated, can run on any node + break; + + case ACTIVE: + //can only run on specific SSI + retVal = checkMatchingSsis(node, statefulTaskSsi); + break; - if (node instanceof SpotinstSlave) { - SpotinstSlave slave = (SpotinstSlave) node; - String ssi = slave.getSsiId(); - boolean isStatefulSlave = StringUtils.isNotEmpty(ssi); - - if (isStatefulSlave) { - boolean canNodeTakeTask = ssi.equals(statefulInterruptedTask.getSsi()); - - if (canNodeTakeTask) { - LOGGER.info("node {} with ssi {} can take task {}", slave.getNodeName(), ssi, - statefulInterruptedTask); - } - else { - retVal = new InconsistentSsiCauseOfBlockage(); - } + case RECYCLE: + case RECYCLING: + case RESUME: + case RESUMING: + default: + //await for SSI to be active + retVal = new AwaitingSsiCauseOfBlockage(); + break; + } + } + } + + return retVal; + } + //endregion + + //region private methods + private CauseOfBlockage checkMatchingSsis(Node node, String statefulTaskSsi) { + CauseOfBlockage retVal = null; + + if (node instanceof SpotinstSlave) { + SpotinstSlave slave = (SpotinstSlave) node; + String ssi = slave.getSsiId(); + boolean isStatefulSlave = StringUtils.isNotEmpty(ssi); + + if (isStatefulSlave) { + boolean canNodeTakeTask = ssi.equals(statefulTaskSsi); + + if (canNodeTakeTask) { + LOGGER.info("node {} with ssi {} can take task", slave.getNodeName(), ssi); } else { retVal = new InconsistentSsiCauseOfBlockage(); @@ -49,16 +91,25 @@ public CauseOfBlockage canTake(Node node, Queue.Task task) { retVal = new InconsistentSsiCauseOfBlockage(); } } + else { + retVal = new InconsistentSsiCauseOfBlockage(); + } return retVal; } - //endregion //region classes private static class InconsistentSsiCauseOfBlockage extends CauseOfBlockage { @Override public String getShortDescription() { - return "task reserved for slave with specific ssi"; + return "task reserved for slave with ssi different from current node"; + } + } + + private static class AwaitingSsiCauseOfBlockage extends CauseOfBlockage { + @Override + public String getShortDescription() { + return "awaiting for stateful node to recover"; } } //endregion diff --git a/src/main/java/hudson/plugins/spotinst/queue/StatefulQueueDecisionHandler.java b/src/main/java/hudson/plugins/spotinst/queue/StatefulQueueDecisionHandler.java index 98b9b1f8..a85516aa 100644 --- a/src/main/java/hudson/plugins/spotinst/queue/StatefulQueueDecisionHandler.java +++ b/src/main/java/hudson/plugins/spotinst/queue/StatefulQueueDecisionHandler.java @@ -4,12 +4,17 @@ import hudson.model.Action; import hudson.model.Node; import hudson.model.Queue; +import hudson.plugins.spotinst.model.aws.AwsStatefulInstance; +import hudson.plugins.spotinst.model.aws.AwsStatefulInstancesManager; import hudson.plugins.spotinst.slave.SpotinstSlave; import jenkins.model.Jenkins; import java.util.List; import java.util.Optional; +/** + * Created by sitay on 30/08/2013. + */ @Extension public class StatefulQueueDecisionHandler extends Queue.QueueDecisionHandler { //region override method @@ -20,20 +25,33 @@ public boolean shouldSchedule(Queue.Task task, List actions) { if (task instanceof StatefulInterruptedTask) { StatefulInterruptedTask statefulInterruptedTask = (StatefulInterruptedTask) task; String ssi = statefulInterruptedTask.getSsi(); - Jenkins jenkinsInstance = Jenkins.getInstanceOrNull(); + AwsStatefulInstance matchingStatefulInstance = AwsStatefulInstancesManager.getStatefulInstanceBySSi(ssi); + boolean isSsiStillExist = matchingStatefulInstance != null; - if (jenkinsInstance != null) { - List nodes = jenkinsInstance.getNodes(); - Optional optionalSsi = - nodes.stream().filter(node -> node instanceof SpotinstSlave).map(node -> (SpotinstSlave) node) - .filter(spotinstSlave -> ssi.equals(spotinstSlave.getSsiId())).findFirst(); + if (isSsiStillExist) { + switch (matchingStatefulInstance.getState()) { + case PAUSE: + case PAUSING: + case PAUSED: + case DEALLOCATE: + case DEALLOCATING: + case DEALLOCATED: + case ERROR: + //Ssi isn't activated, no need to wait for it + break; - if (optionalSsi.isPresent()) { - SpotinstSlave matchedSlave = optionalSsi.get(); - retVal = matchedSlave.isAcceptingTasks(); - } - else { - retVal = false; + case ACTIVE: + retVal = isActiveStatefulInstanceAcceptingTasks(ssi); + break; + + case RECYCLE: + case RECYCLING: + case RESUME: + case RESUMING: + default: + //await for ssi to be active + retVal = false; + break; } } } @@ -41,4 +59,25 @@ public boolean shouldSchedule(Queue.Task task, List actions) { return retVal; } //endregion + + //region private methods + private Boolean isActiveStatefulInstanceAcceptingTasks(String ssi) { + boolean retVal = false; + + Jenkins jenkinsInstance = Jenkins.getInstanceOrNull(); + + if (jenkinsInstance != null) { + List nodes = jenkinsInstance.getNodes(); + Optional optionalSsi = + nodes.stream().filter(node -> node instanceof SpotinstSlave).map(node -> (SpotinstSlave) node) + .filter(spotinstSlave -> ssi.equals(spotinstSlave.getSsiId())).findFirst(); + + if (optionalSsi.isPresent()) { + SpotinstSlave matchedSlave = optionalSsi.get(); + retVal = matchedSlave.isAcceptingTasks(); + } + } + + return retVal; + } } diff --git a/src/test/java/hudson/plugins/spotinst/SpotinstCloudTest.java b/src/test/java/hudson/plugins/spotinst/SpotinstCloudTest.java index 5f7b22e4..d08a6816 100644 --- a/src/test/java/hudson/plugins/spotinst/SpotinstCloudTest.java +++ b/src/test/java/hudson/plugins/spotinst/SpotinstCloudTest.java @@ -1275,7 +1275,9 @@ public void testSpotinstSlaveTermination_ifAgentInPendingInstances_thenAgentIsRe Mockito.when( RepoManager.getInstance().getAwsGroupRepo().getGroupInstances(Mockito.anyString(), Mockito.anyString())) .thenReturn(new ApiResponse<>(result)); - + Mockito.when( + RepoManager.getInstance().getAwsGroupRepo().getStatefulInstances(Mockito.anyString(), Mockito.anyString())) + .thenReturn(new ApiResponse<>(new LinkedList<>())); List spots = Collections.singletonList(newSpot); AwsScaleUpResult scaleUpResult = new AwsScaleUpResult(); From fddd46e25c9e0672dee334d884b575da6eaa8a82 Mon Sep 17 00:00:00 2001 From: sitay Date: Thu, 31 Aug 2023 23:34:26 +0300 Subject: [PATCH 07/13] initial commit --- .../plugins/spotinst/model/aws/AwsStatefulInstancesManager.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java index afe8822d..98f9a092 100644 --- a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java +++ b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java @@ -30,6 +30,7 @@ public static AwsStatefulInstance getStatefulInstanceBySSi(String ssi){ return retVal; } + //endregion //region getters & setters public static Map> getAwsStatefulInstanceBySsiById() { From fab86040b6a8a66ee799d11f159835f4801a80c7 Mon Sep 17 00:00:00 2001 From: sitay Date: Thu, 31 Aug 2023 23:40:58 +0300 Subject: [PATCH 08/13] initial commit --- .../java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java index 6d88ab65..ae98359f 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java @@ -100,12 +100,12 @@ List scaleUp(ProvisionRequest request) { @Override protected String getStatefulInstanceId(String instanceId) { - return null;//TODO: implement + return null; } @Override public Boolean deallocateInstance(String instanceId){ - return false;//TODO: implement + return false; } @Override From 1ca232be1f06406181dbd8d8621eeff02af60754 Mon Sep 17 00:00:00 2001 From: sitay Date: Wed, 6 Sep 2023 11:31:03 +0300 Subject: [PATCH 09/13] initial commit --- .../plugins/spotinst/api/SpotinstApi.java | 10 +- .../spotinst/cloud/AwsSpotinstCloud.java | 4 +- .../spotinst/cloud/BaseSpotinstCloud.java | 15 +- .../AwsDeallocateStatefulInstanceRequest.java | 23 ++ .../aws/AwsStatefulDeallocationConfig.java | 50 ++++ .../aws/AwsStatefulInstancesManager.java | 62 ++++- .../queue/SpotQueueTaskDispatcher.java | 232 +++++++++--------- .../spotinst/queue/SsiByTaskMapper.java | 31 --- .../queue/StatefulInterruptedTask.java | 96 +++++++- .../queue/StatefulQueueDecisionHandler.java | 83 ------- .../spotinst/slave/SpotLauncherHelper.java | 43 ++-- .../spotinst/slave/SpotinstComputer.java | 54 ++-- .../plugins/spotinst/slave/SpotinstSlave.java | 96 ++++++++ 13 files changed, 515 insertions(+), 284 deletions(-) create mode 100644 src/main/java/hudson/plugins/spotinst/model/aws/AwsDeallocateStatefulInstanceRequest.java create mode 100644 src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulDeallocationConfig.java delete mode 100644 src/main/java/hudson/plugins/spotinst/queue/SsiByTaskMapper.java delete mode 100644 src/main/java/hudson/plugins/spotinst/queue/StatefulQueueDecisionHandler.java diff --git a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java index 32536b06..7dc6b058 100644 --- a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java +++ b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java @@ -153,9 +153,13 @@ public static Boolean awsDeallocateInstance(String groupId, String statefulInsta Map headers = buildHeaders(); Map queryParams = buildQueryParams(accountId); - AwsDetachInstancesRequest request = new AwsDetachInstancesRequest(); - request.setShouldDecrementTargetCapacity(true); - request.setShouldTerminateInstances(true); + AwsDeallocateStatefulInstanceRequest request = new AwsDeallocateStatefulInstanceRequest(); + AwsStatefulDeallocationConfig statefulDeallocation = new AwsStatefulDeallocationConfig(); + statefulDeallocation.setShouldDeleteImages(true); + statefulDeallocation.setShouldDeleteSnapshots(true); + statefulDeallocation.setShouldDeleteVolumes(true); + statefulDeallocation.setShouldDeleteNetworkInterfaces(true); + request.setStatefulDeallocation(statefulDeallocation); String body = JsonMapper.toJson(request); diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java index d0f8028e..cb8d4662 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java @@ -279,7 +279,7 @@ private void syncGroupStatefulInstances() { Map ssiById = new HashMap<>(); statefulInstances.forEach(ssi -> ssiById.put(ssi.getId(), ssi)); - AwsStatefulInstancesManager.getAwsStatefulInstanceBySsiById().put(groupId, ssiById); + AwsStatefulInstancesManager.getAwsStatefulInstanceBySsiByGroupId().put(groupId, ssiById); } private List handleNewAwsSpots(AwsScaleUpResult scaleUpResult, String label) { @@ -429,7 +429,7 @@ private Boolean isSlaveExistForInstance(AwsGroupInstance instance) { private String getSsiByInstance(String instanceId) { String retVal = null; Map ssiById = - AwsStatefulInstancesManager.getAwsStatefulInstanceBySsiById().get(groupId); + AwsStatefulInstancesManager.getAwsStatefulInstanceBySsiByGroupId().get(groupId); if(ssiById != null) { Optional optionalMatchingSsi = diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index b910c0fb..92dc9d72 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -759,7 +759,16 @@ public void setSpotReTriggerBuilds(SpotReTriggerBuilds spotReTriggerBuilds) { } public Boolean getShouldReTriggerBuilds() { - return getSpotReTriggerBuilds().getShouldReTriggerBuilds(); + Boolean retVal; + + if(getSpotReTriggerBuilds() == null){ + retVal = null; + } + else{ + retVal = getSpotReTriggerBuilds().getShouldReTriggerBuilds(); + } + + return retVal; } public void setShouldReTriggerBuilds(Boolean shouldReTriggerBuilds) { @@ -767,6 +776,10 @@ public void setShouldReTriggerBuilds(Boolean shouldReTriggerBuilds) { } public Boolean getStickyNode() { + if(getSpotReTriggerBuilds() == null){ + return false; + } + return getSpotReTriggerBuilds().getStickyNode(); } diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/AwsDeallocateStatefulInstanceRequest.java b/src/main/java/hudson/plugins/spotinst/model/aws/AwsDeallocateStatefulInstanceRequest.java new file mode 100644 index 00000000..3fc79180 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/model/aws/AwsDeallocateStatefulInstanceRequest.java @@ -0,0 +1,23 @@ +package hudson.plugins.spotinst.model.aws; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * Created by sitay on 30/08/23. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class AwsDeallocateStatefulInstanceRequest { + //region members + private AwsStatefulDeallocationConfig statefulDeallocation; + //endregion + + //region getters & setters + public AwsStatefulDeallocationConfig getStatefulDeallocation() { + return statefulDeallocation; + } + + public void setStatefulDeallocation(AwsStatefulDeallocationConfig statefulDeallocation) { + this.statefulDeallocation = statefulDeallocation; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulDeallocationConfig.java b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulDeallocationConfig.java new file mode 100644 index 00000000..507fa01b --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulDeallocationConfig.java @@ -0,0 +1,50 @@ +package hudson.plugins.spotinst.model.aws; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * Created by sitay on 30/08/23. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class AwsStatefulDeallocationConfig { + //region members + private Boolean shouldDeleteImages; + private Boolean shouldDeleteNetworkInterfaces; + private Boolean shouldDeleteVolumes; + private Boolean shouldDeleteSnapshots; + //endregion + + //region getters & setters + public Boolean getShouldDeleteImages() { + return shouldDeleteImages; + } + + public void setShouldDeleteImages(Boolean shouldDeleteImages) { + this.shouldDeleteImages = shouldDeleteImages; + } + + public Boolean getShouldDeleteNetworkInterfaces() { + return shouldDeleteNetworkInterfaces; + } + + public void setShouldDeleteNetworkInterfaces(Boolean shouldDeleteNetworkInterfaces) { + this.shouldDeleteNetworkInterfaces = shouldDeleteNetworkInterfaces; + } + + public Boolean getShouldDeleteVolumes() { + return shouldDeleteVolumes; + } + + public void setShouldDeleteVolumes(Boolean shouldDeleteVolumes) { + this.shouldDeleteVolumes = shouldDeleteVolumes; + } + + public Boolean getShouldDeleteSnapshots() { + return shouldDeleteSnapshots; + } + + public void setShouldDeleteSnapshots(Boolean shouldDeleteSnapshots) { + this.shouldDeleteSnapshots = shouldDeleteSnapshots; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java index 98f9a092..191feb8d 100644 --- a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java +++ b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java @@ -1,8 +1,9 @@ package hudson.plugins.spotinst.model.aws; -import java.util.Collection; -import java.util.Map; -import java.util.Optional; +import hudson.model.Executor; +import hudson.model.Queue; + +import java.util.*; import java.util.concurrent.ConcurrentHashMap; /** @@ -10,15 +11,30 @@ */ public class AwsStatefulInstancesManager { //region members - private final static Map> awsStatefulInstanceBySsiById = + private static final Map ssiByStatefulTask = + new ConcurrentHashMap<>(); + private static final Map> awsStatefulInstanceBySsiByGroupId = new ConcurrentHashMap<>(); + private static final Set statefulTasksToReTrigger = + new HashSet<>(); //endregion //region methods - public static AwsStatefulInstance getStatefulInstanceBySSi(String ssi){ - AwsStatefulInstance retVal = null; - Collection> allGroupsSsiById = - awsStatefulInstanceBySsiById.values(); + public static void putSsiByTask(Queue.Task task, Executor executor, String ssiId) { + String key = generateKey(task, executor); + ssiByStatefulTask.put(key, ssiId); + } + + public static String removeSsiByTask(Queue.Task task, Executor executor) { + String retVal; + String key = generateKey(task, executor); + retVal = ssiByStatefulTask.remove(key); + return retVal; + } + + public static AwsStatefulInstance getStatefulInstanceBySSi(String ssi) { + AwsStatefulInstance retVal = null; + Collection> allGroupsSsiById = awsStatefulInstanceBySsiByGroupId.values(); Optional optionalMatchingStatefulInstance = allGroupsSsiById.stream().filter(groupSsiById -> groupSsiById.containsKey(ssi)) @@ -30,11 +46,37 @@ public static AwsStatefulInstance getStatefulInstanceBySSi(String ssi){ return retVal; } + + + public static Boolean isReTriggeringStatefulTask(Queue.Task task, Executor executor) { + boolean retVal; + String key = generateKey(task, executor); + retVal = statefulTasksToReTrigger.contains(key); + return retVal; + } + + public static void handleReTriggeringStatefulTask(Queue.Task task, Executor executor) { + String key = generateKey(task, executor); + statefulTasksToReTrigger.add(key); + } + + public static Boolean handleReTriggeredStatefulTask(Queue.Task task, Executor executor) { + boolean retVal; + String key = generateKey(task, executor); + retVal = statefulTasksToReTrigger.remove(key); + return retVal; + } + //endregion + + //region private methods + private static String generateKey(Queue.Task task, Executor executor) { + return String.format("%s_%s", task, executor.getId()); + } //endregion //region getters & setters - public static Map> getAwsStatefulInstanceBySsiById() { - return awsStatefulInstanceBySsiById; + public static Map> getAwsStatefulInstanceBySsiByGroupId() { + return awsStatefulInstanceBySsiByGroupId; } //endregion } diff --git a/src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java b/src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java index 5d62ff34..959bf42a 100644 --- a/src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java +++ b/src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java @@ -1,116 +1,116 @@ -package hudson.plugins.spotinst.queue; - -import hudson.Extension; -import hudson.model.Node; -import hudson.model.Queue; -import hudson.model.queue.CauseOfBlockage; -import hudson.model.queue.QueueTaskDispatcher; -import hudson.plugins.spotinst.model.aws.AwsStatefulInstance; -import hudson.plugins.spotinst.model.aws.AwsStatefulInstancesManager; -import hudson.plugins.spotinst.slave.SpotinstSlave; -import org.apache.commons.lang.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - -/** - * Created by sitay on 30/08/2013. - */ -@Extension -public class SpotQueueTaskDispatcher extends QueueTaskDispatcher { - //region members - private static final Logger LOGGER = LoggerFactory.getLogger(SpotQueueTaskDispatcher.class); - //endregion - - //region override methods - @Override - public CauseOfBlockage canTake(Node node, Queue.Task task) { - CauseOfBlockage retVal = null; - boolean isTaskReservedForStatefulSlave = task instanceof StatefulInterruptedTask; - - if (isTaskReservedForStatefulSlave) { - StatefulInterruptedTask statefulInterruptedTask = (StatefulInterruptedTask) task; - String statefulTaskSsi = statefulInterruptedTask.getSsi(); - AwsStatefulInstance matchingStatefulInstance = - AwsStatefulInstancesManager.getStatefulInstanceBySSi(statefulTaskSsi); - boolean isSsiExist = matchingStatefulInstance != null; - - if (isSsiExist) { - switch (matchingStatefulInstance.getState()) { - case PAUSE: - case PAUSING: - case PAUSED: - case DEALLOCATE: - case DEALLOCATING: - case DEALLOCATED: - case ERROR: - //SSI isn't activated, can run on any node - break; - - case ACTIVE: - //can only run on specific SSI - retVal = checkMatchingSsis(node, statefulTaskSsi); - break; - - case RECYCLE: - case RECYCLING: - case RESUME: - case RESUMING: - default: - //await for SSI to be active - retVal = new AwaitingSsiCauseOfBlockage(); - break; - } - } - } - - return retVal; - } - //endregion - - //region private methods - private CauseOfBlockage checkMatchingSsis(Node node, String statefulTaskSsi) { - CauseOfBlockage retVal = null; - - if (node instanceof SpotinstSlave) { - SpotinstSlave slave = (SpotinstSlave) node; - String ssi = slave.getSsiId(); - boolean isStatefulSlave = StringUtils.isNotEmpty(ssi); - - if (isStatefulSlave) { - boolean canNodeTakeTask = ssi.equals(statefulTaskSsi); - - if (canNodeTakeTask) { - LOGGER.info("node {} with ssi {} can take task", slave.getNodeName(), ssi); - } - else { - retVal = new InconsistentSsiCauseOfBlockage(); - } - } - else { - retVal = new InconsistentSsiCauseOfBlockage(); - } - } - else { - retVal = new InconsistentSsiCauseOfBlockage(); - } - - return retVal; - } - - //region classes - private static class InconsistentSsiCauseOfBlockage extends CauseOfBlockage { - @Override - public String getShortDescription() { - return "task reserved for slave with ssi different from current node"; - } - } - - private static class AwaitingSsiCauseOfBlockage extends CauseOfBlockage { - @Override - public String getShortDescription() { - return "awaiting for stateful node to recover"; - } - } - //endregion -} +//package hudson.plugins.spotinst.queue; +// +//import hudson.Extension; +//import hudson.model.Node; +//import hudson.model.Queue; +//import hudson.model.queue.CauseOfBlockage; +//import hudson.model.queue.QueueTaskDispatcher; +//import hudson.plugins.spotinst.model.aws.AwsStatefulInstance; +//import hudson.plugins.spotinst.model.aws.AwsStatefulInstancesManager; +//import hudson.plugins.spotinst.slave.SpotinstSlave; +//import org.apache.commons.lang.StringUtils; +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; +// +// +///** +// * Created by sitay on 30/08/2013. +// */ +//@Extension +//public class SpotQueueTaskDispatcher extends QueueTaskDispatcher { +// //region members +// private static final Logger LOGGER = LoggerFactory.getLogger(SpotQueueTaskDispatcher.class); +// //endregion +// +// //region override methods +// @Override +// public CauseOfBlockage canTake(Node node, Queue.Task task) { +// CauseOfBlockage retVal = null; +// boolean isTaskReservedForStatefulSlave = task instanceof StatefulInterruptedTask; +// +// if (isTaskReservedForStatefulSlave) { +// StatefulInterruptedTask statefulInterruptedTask = (StatefulInterruptedTask) task; +// String statefulTaskSsi = statefulInterruptedTask.getSsi(); +// AwsStatefulInstance matchingStatefulInstance = +// AwsStatefulInstancesManager.getStatefulInstanceBySSi(statefulTaskSsi); +// boolean isSsiExist = matchingStatefulInstance != null; +// +// if (isSsiExist) { +// switch (matchingStatefulInstance.getState()) { +// case PAUSE: +// case PAUSING: +// case PAUSED: +// case DEALLOCATE: +// case DEALLOCATING: +// case DEALLOCATED: +// case ERROR: +// //SSI isn't activated, can run on any node +// break; +// +// case ACTIVE: +// //can only run on specific SSI +// retVal = checkMatchingSsis(node, statefulTaskSsi); +// break; +// +// case RECYCLE: +// case RECYCLING: +// case RESUME: +// case RESUMING: +// default: +// //await for SSI to be active +// retVal = new AwaitingSsiCauseOfBlockage(); +// break; +// } +// } +// } +// +// return retVal; +// } +// //endregion +// +// //region private methods +// private CauseOfBlockage checkMatchingSsis(Node node, String statefulTaskSsi) { +// CauseOfBlockage retVal = null; +// +// if (node instanceof SpotinstSlave) { +// SpotinstSlave slave = (SpotinstSlave) node; +// String ssi = slave.getSsiId(); +// boolean isStatefulSlave = StringUtils.isNotEmpty(ssi); +// +// if (isStatefulSlave) { +// boolean canNodeTakeTask = ssi.equals(statefulTaskSsi); +// +// if (canNodeTakeTask) { +// LOGGER.info("node {} with ssi {} can take task", slave.getNodeName(), ssi); +// } +// else { +// retVal = new InconsistentSsiCauseOfBlockage(); +// } +// } +// else { +// retVal = new InconsistentSsiCauseOfBlockage(); +// } +// } +// else { +// retVal = new InconsistentSsiCauseOfBlockage(); +// } +// +// return retVal; +// } +// +// //region classes +// private static class InconsistentSsiCauseOfBlockage extends CauseOfBlockage { +// @Override +// public String getShortDescription() { +// return "task reserved for slave with ssi different from current node"; +// } +// } +// +// private static class AwaitingSsiCauseOfBlockage extends CauseOfBlockage { +// @Override +// public String getShortDescription() { +// return "awaiting for stateful node to recover"; +// } +// } +// //endregion +//} diff --git a/src/main/java/hudson/plugins/spotinst/queue/SsiByTaskMapper.java b/src/main/java/hudson/plugins/spotinst/queue/SsiByTaskMapper.java deleted file mode 100644 index d448973e..00000000 --- a/src/main/java/hudson/plugins/spotinst/queue/SsiByTaskMapper.java +++ /dev/null @@ -1,31 +0,0 @@ -package hudson.plugins.spotinst.queue; - -import hudson.model.Executor; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -public class SsiByTaskMapper { - //region members - private static final Map ssiByStatefulTask = new ConcurrentHashMap<>(); - //endregion - - //region methods - public static void putSsiByTask(String taskName, Executor executor, String ssiId){ - String key = generateKey(taskName, executor); - ssiByStatefulTask.put(key, ssiId); - } - - public static String removeSsiByTask(String taskName, Executor executor){ - String retVal; - String key = generateKey(taskName, executor); - retVal = ssiByStatefulTask.remove(key); - return retVal; - } - //endregion - - //region private methods - private static String generateKey(String taskName, Executor executor){ - return String.format("%s_%s", taskName, executor.getId()); - } -} diff --git a/src/main/java/hudson/plugins/spotinst/queue/StatefulInterruptedTask.java b/src/main/java/hudson/plugins/spotinst/queue/StatefulInterruptedTask.java index bda03630..5719c84a 100644 --- a/src/main/java/hudson/plugins/spotinst/queue/StatefulInterruptedTask.java +++ b/src/main/java/hudson/plugins/spotinst/queue/StatefulInterruptedTask.java @@ -1,10 +1,17 @@ package hudson.plugins.spotinst.queue; +import hudson.model.Label; +import hudson.model.Node; import hudson.model.Queue; +import hudson.model.ResourceList; +import hudson.model.queue.CauseOfBlockage; +import hudson.model.queue.SubTask; +import org.acegisecurity.Authentication; import javax.annotation.CheckForNull; import javax.annotation.Nonnull; import java.io.IOException; +import java.util.Collection; public class StatefulInterruptedTask implements Queue.Task { //region members @@ -19,7 +26,7 @@ public StatefulInterruptedTask(@Nonnull String ssi, @Nonnull Queue.Task task) { this.ssi = ssi; if(task instanceof StatefulInterruptedTask){ - this.task = ((StatefulInterruptedTask) task).task; + this.task = ((StatefulInterruptedTask) task).getTask(); } else { this.task = task; @@ -33,29 +40,92 @@ public String getName() { return task.getName(); } + @Override + public ResourceList getResourceList() { + return task.getResourceList(); + } + + @Override + public String getDisplayName() { + return task.getDisplayName(); + } + @Override public String getFullDisplayName() { return task.getFullDisplayName(); } + @Override + public String getUrl() { + return task.getUrl(); + } + + @Override + public boolean isConcurrentBuild() { + return task.isConcurrentBuild(); + } + + @Override + public Collection getSubTasks() { + return task.getSubTasks(); + } + + @Override + public String getAffinityKey() { + return task.getAffinityKey(); + } + @Override public void checkAbortPermission() { task.checkAbortPermission(); } + @Override + public CauseOfBlockage getCauseOfBlockage() { + return task.getCauseOfBlockage(); + } + + @Override + public boolean isBuildBlocked() { + return task.isBuildBlocked(); + } + + @Override + public String getWhyBlocked() { + return task.getWhyBlocked(); + } + + @Nonnull + @Override + public Authentication getDefaultAuthentication() { + return task.getDefaultAuthentication(); + } + + @Nonnull + @Override + public Authentication getDefaultAuthentication(Queue.Item item) { + return task.getDefaultAuthentication(item); + } + @Override public boolean hasAbortPermission() { return task.hasAbortPermission(); } + @Override - public String getUrl() { - return task.getUrl(); + public Label getAssignedLabel() { + return task.getAssignedLabel(); } @Override - public String getDisplayName() { - return task.getDisplayName(); + public Node getLastBuiltOn() { + return task.getLastBuiltOn(); + } + + @Override + public long getEstimatedDuration() { + return task.getEstimatedDuration(); } @CheckForNull @@ -63,9 +133,25 @@ public String getDisplayName() { public Queue.Executable createExecutable() throws IOException { return task.createExecutable(); } + + @Nonnull + @Override + public Queue.Task getOwnerTask() { + return task.getOwnerTask(); + } + + @Override + public Object getSameNodeConstraint() { + return task.getSameNodeConstraint(); + } //endregion //region getters & setters + @Nonnull + public Queue.Task getTask() { + return task; + } + @Nonnull public String getSsi() { return ssi; diff --git a/src/main/java/hudson/plugins/spotinst/queue/StatefulQueueDecisionHandler.java b/src/main/java/hudson/plugins/spotinst/queue/StatefulQueueDecisionHandler.java deleted file mode 100644 index a85516aa..00000000 --- a/src/main/java/hudson/plugins/spotinst/queue/StatefulQueueDecisionHandler.java +++ /dev/null @@ -1,83 +0,0 @@ -package hudson.plugins.spotinst.queue; - -import hudson.Extension; -import hudson.model.Action; -import hudson.model.Node; -import hudson.model.Queue; -import hudson.plugins.spotinst.model.aws.AwsStatefulInstance; -import hudson.plugins.spotinst.model.aws.AwsStatefulInstancesManager; -import hudson.plugins.spotinst.slave.SpotinstSlave; -import jenkins.model.Jenkins; - -import java.util.List; -import java.util.Optional; - -/** - * Created by sitay on 30/08/2013. - */ -@Extension -public class StatefulQueueDecisionHandler extends Queue.QueueDecisionHandler { - //region override method - @Override - public boolean shouldSchedule(Queue.Task task, List actions) { - boolean retVal = true; - - if (task instanceof StatefulInterruptedTask) { - StatefulInterruptedTask statefulInterruptedTask = (StatefulInterruptedTask) task; - String ssi = statefulInterruptedTask.getSsi(); - AwsStatefulInstance matchingStatefulInstance = AwsStatefulInstancesManager.getStatefulInstanceBySSi(ssi); - boolean isSsiStillExist = matchingStatefulInstance != null; - - if (isSsiStillExist) { - switch (matchingStatefulInstance.getState()) { - case PAUSE: - case PAUSING: - case PAUSED: - case DEALLOCATE: - case DEALLOCATING: - case DEALLOCATED: - case ERROR: - //Ssi isn't activated, no need to wait for it - break; - - case ACTIVE: - retVal = isActiveStatefulInstanceAcceptingTasks(ssi); - break; - - case RECYCLE: - case RECYCLING: - case RESUME: - case RESUMING: - default: - //await for ssi to be active - retVal = false; - break; - } - } - } - - return retVal; - } - //endregion - - //region private methods - private Boolean isActiveStatefulInstanceAcceptingTasks(String ssi) { - boolean retVal = false; - - Jenkins jenkinsInstance = Jenkins.getInstanceOrNull(); - - if (jenkinsInstance != null) { - List nodes = jenkinsInstance.getNodes(); - Optional optionalSsi = - nodes.stream().filter(node -> node instanceof SpotinstSlave).map(node -> (SpotinstSlave) node) - .filter(spotinstSlave -> ssi.equals(spotinstSlave.getSsiId())).findFirst(); - - if (optionalSsi.isPresent()) { - SpotinstSlave matchedSlave = optionalSsi.get(); - retVal = matchedSlave.isAcceptingTasks(); - } - } - - return retVal; - } -} diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java b/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java index 5607b035..2f694e73 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java @@ -1,16 +1,16 @@ package hudson.plugins.spotinst.slave; import hudson.model.*; +import hudson.model.Queue; import hudson.model.queue.SubTask; -import hudson.plugins.spotinst.queue.SsiByTaskMapper; +import hudson.plugins.spotinst.model.aws.AwsStatefulInstancesManager; import hudson.plugins.spotinst.queue.StatefulInterruptedTask; import hudson.slaves.SlaveComputer; import org.apache.commons.lang.BooleanUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.LinkedList; -import java.util.List; +import java.util.*; /** @@ -21,6 +21,7 @@ class SpotLauncherHelper { //region Members private static final Logger LOGGER = LoggerFactory.getLogger(SpotLauncherHelper.class); + private static final Object lock = new Object(); //endregion //region Methods @@ -33,7 +34,6 @@ static void handleDisconnect(final SlaveComputer computer, Boolean shouldRetrigg SpotinstComputer spotinstComputer = (SpotinstComputer) computer; SpotinstSlave slave = spotinstComputer.getNode(); - if (shouldRetriggerBuilds && (slave == null || BooleanUtils.isFalse(slave.isSlavePending()))) { LOGGER.info(String.format("Start retriggering executors for %s", spotinstComputer.getDisplayName())); @@ -54,19 +54,30 @@ static void handleDisconnect(final SlaveComputer computer, Boolean shouldRetrigg actions = ((Actionable) executable).getActions(); } - String ssiByTaskName = SsiByTaskMapper.removeSsiByTask(task.getName(), executor); - - if (ssiByTaskName != null) { - StatefulInterruptedTask statefulInterruptedTask = - new StatefulInterruptedTask(ssiByTaskName, task); - LOGGER.info(String.format("RETRIGGERING Stateful Task: %s - WITH ACTIONS: %s", statefulInterruptedTask, actions)); - - Queue.getInstance().schedule2(statefulInterruptedTask, 10, actions); - } - else { - LOGGER.info(String.format("RETRIGGERING: %s - WITH ACTIONS: %s", task, actions)); - Queue.getInstance().schedule2(task, 10, actions); + synchronized (lock) { + String ssiByTaskName = AwsStatefulInstancesManager.removeSsiByTask(task, executor); + + if (ssiByTaskName != null) { + AwsStatefulInstancesManager.handleReTriggeringStatefulTask(task, executor); + StatefulInterruptedTask statefulInterruptedTask = + new StatefulInterruptedTask(ssiByTaskName, task); + LOGGER.info(String.format("RETRIGGERING Stateful Task: %s - WITH ACTIONS: %s on SSI %s", + statefulInterruptedTask.getTask(), actions, ssiByTaskName)); + + Queue.getInstance().schedule2(statefulInterruptedTask, 10, actions); + } + else { + if (AwsStatefulInstancesManager.isReTriggeringStatefulTask(task, executor)) { + LOGGER.info(String.format( + "Stateful Task : %s is already queued for re-triggering - ignoring it", + task)); + } + else { + LOGGER.info(String.format("RETRIGGERING: %s - WITH ACTIONS: %s", task, actions)); + Queue.getInstance().schedule2(task, 10, actions); + } + } } } } diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java index 2902b7ad..21aabe33 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java @@ -4,8 +4,7 @@ import hudson.model.Node; import hudson.model.Queue; import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; -import hudson.plugins.spotinst.queue.SsiByTaskMapper; -import hudson.plugins.spotinst.queue.StatefulInterruptedTask; +import hudson.plugins.spotinst.model.aws.AwsStatefulInstancesManager; import hudson.slaves.SlaveComputer; import org.apache.commons.lang.StringUtils; import org.kohsuke.stapler.HttpRedirect; @@ -49,22 +48,17 @@ public void taskAccepted(Executor executor, Queue.Task task) { this.setTemporarilyOffline(true, spotinstSingleTaskOfflineCause); } - if(spotinstCloud.getStickyNode()) { + if (spotinstCloud.getStickyNode()) { String spotinstNodeSsiId = spotinstNode.getSsiId(); if (StringUtils.isNotEmpty(spotinstNodeSsiId)) { - SsiByTaskMapper.putSsiByTask(task.getName(), executor, spotinstNodeSsiId); - - //TODO: remove - only for logs - if (task instanceof StatefulInterruptedTask) { - StatefulInterruptedTask statefulInterruptedTask = (StatefulInterruptedTask) task; - String statefulTaskSsiId = statefulInterruptedTask.getSsi(); - - if (spotinstNodeSsiId.equals(statefulTaskSsiId) == false) { - LOGGER.warn("stateful task is reserved for ssi {}, however it is running on ssi {}", - statefulTaskSsiId, spotinstNodeSsiId); - } - } + LOGGER.info("task {} accepted on executor {} and is bound to ssi {}", task, executor.getId(), + spotinstNodeSsiId); + AwsStatefulInstancesManager.putSsiByTask(task, executor, spotinstNodeSsiId); + } + else { + LOGGER.info("task {} accepted on a stateless slave {}. no restrictions occur", task.getName(), + spotinstNode.getNodeName()); } } } @@ -82,9 +76,35 @@ public void taskAccepted(Executor executor, Queue.Task task) { } @Override - public void taskCompleted(Executor executor, Queue.Task task, long durationMS){ + public void taskCompleted(Executor executor, Queue.Task task, long durationMS) { super.taskCompleted(executor, task, durationMS); - SsiByTaskMapper.removeSsiByTask(task.getName(), executor); + SpotinstSlave spotinstNode = this.getNode(); + + if (spotinstNode != null) { + BaseSpotinstCloud spotinstCloud = spotinstNode.getSpotinstCloud(); + + if (spotinstCloud != null) { + if (spotinstCloud.getStickyNode()) { + String spotinstNodeSsiId = spotinstNode.getSsiId(); + boolean isStatefulNode = StringUtils.isNotEmpty(spotinstNodeSsiId); + + if (isStatefulNode) { + LOGGER.info("task {} accepted on executor {} and is bound to ssi {}", task, executor.getId(), + spotinstNodeSsiId); + String attachedSsi = AwsStatefulInstancesManager.removeSsiByTask(task, executor); + boolean isTaskReTriggered = StringUtils.isEmpty(attachedSsi); + + if (isTaskReTriggered) { + AwsStatefulInstancesManager.handleReTriggeredStatefulTask(task, executor); + } + else { + LOGGER.info("unbinding stateful task {} and executor {} from ssi {}", task.getName(), + executor.getId(), attachedSsi); + } + } + } + } + } } //endregion diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java index 4f0c9c20..1539d00f 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java @@ -2,10 +2,16 @@ import hudson.Extension; import hudson.model.*; +import hudson.model.queue.CauseOfBlockage; import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; +import hudson.plugins.spotinst.common.StatefulInstanceStateEnum; +import hudson.plugins.spotinst.model.aws.AwsStatefulInstance; +import hudson.plugins.spotinst.model.aws.AwsStatefulInstancesManager; +import hudson.plugins.spotinst.queue.StatefulInterruptedTask; import hudson.slaves.*; import jenkins.model.Jenkins; import net.sf.json.JSONObject; +import org.apache.commons.lang.StringUtils; import org.kohsuke.stapler.StaplerRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -180,6 +186,82 @@ public Node reconfigure(StaplerRequest req, JSONObject form) throws Descriptor.F public Node asNode() { return this; } + + @Override + public CauseOfBlockage canTake(Queue.BuildableItem item){ + CauseOfBlockage retVal = super.canTake(item); + + if(retVal == null) { + LOGGER.info("can node {} take item {}", this, item);//TODO: remove + Queue.Task task = item.task; + boolean isTaskReservedForStatefulSlave = task instanceof StatefulInterruptedTask; + + if (isTaskReservedForStatefulSlave) { + StatefulInterruptedTask statefulInterruptedTask = (StatefulInterruptedTask) task; + String statefulTaskSsi = statefulInterruptedTask.getSsi(); + AwsStatefulInstance matchingStatefulInstance = AwsStatefulInstancesManager.getStatefulInstanceBySSi(statefulTaskSsi); + boolean isSsiExist = matchingStatefulInstance != null; + + if (isSsiExist) { + StatefulInstanceStateEnum ssiState = matchingStatefulInstance.getState(); + + switch (ssiState) { + case PAUSE: + case PAUSING: + case PAUSED: + case DEALLOCATE: + case DEALLOCATING: + case DEALLOCATED: + case ERROR: + LOGGER.info("SSI {} is in state {} and isn't activated, task {} can run on any node", + statefulTaskSsi, ssiState, task.getName()); + break; + + case ACTIVE: + retVal = checkMatchingSsis(statefulTaskSsi); + break; + + case RECYCLE: + case RECYCLING: + case RESUME: + case RESUMING: + default: + LOGGER.info("task {} awaits for SSI {} to be active. no node can take it", task.getName(), + statefulTaskSsi); + retVal = new AwaitingSsiCauseOfBlockage(); + break; + } + } + } + } + + return retVal; + } + + private CauseOfBlockage checkMatchingSsis(String statefulTaskSsi) { + CauseOfBlockage retVal = null; + String slaveSsi = getSsiId(); + boolean isStatefulSlave = StringUtils.isNotEmpty(slaveSsi); + + if (isStatefulSlave) { + boolean canNodeTakeTask = slaveSsi.equals(statefulTaskSsi); + + if (canNodeTakeTask) { + LOGGER.info("node {} with ssi {} can take task", getNodeName(), slaveSsi); + } + else { + LOGGER.info("task reserved for stateful slave with SSI {} different from current slave's SSI {}", + statefulTaskSsi, slaveSsi); + retVal = new InconsistentSsiCauseOfBlockage(); + } + } + else { + LOGGER.info("task reserved for stateful slave with SSI {}. current slave is not a stateful", statefulTaskSsi); + retVal = new InconsistentSsiCauseOfBlockage(); + } + + return retVal; + } //endregion //region Public Methods @@ -278,5 +360,19 @@ public boolean isInstantiable() { return false; } } + + private static class InconsistentSsiCauseOfBlockage extends CauseOfBlockage { + @Override + public String getShortDescription() { + return "task reserved for slave with ssi different from current node"; + } + } + + private static class AwaitingSsiCauseOfBlockage extends CauseOfBlockage { + @Override + public String getShortDescription() { + return "awaiting for stateful node to recover"; + } + } //endregion } From f4c8a26ccffeffd76c924710660d93067fd7c869 Mon Sep 17 00:00:00 2001 From: sitay Date: Tue, 26 Sep 2023 17:13:57 +0300 Subject: [PATCH 10/13] merge deallocation --- .../plugins/spotinst/api/SpotinstApi.java | 6 +- .../spotinst/cloud/AwsSpotinstCloud.java | 125 ++++------- .../spotinst/cloud/AzureSpotCloud.java | 7 +- .../spotinst/cloud/AzureSpotinstCloud.java | 7 +- .../spotinst/cloud/BaseSpotinstCloud.java | 42 ++-- .../spotinst/cloud/GcpSpotinstCloud.java | 7 +- .../common/StatefulInstanceStateEnum.java | 62 ------ ...java => AwsStatefulInstanceStateEnum.java} | 12 +- .../stateful/StatefulInstanceManager.java | 195 ++++++++++++++++++ .../aws/AwsStatefulDeallocationConfig.java | 50 ----- .../model/aws/AwsStatefulInstance.java | 44 ---- .../aws/AwsStatefulInstancesManager.java | 82 -------- .../aws/AwsStatefulInstancesResponse.java | 6 - .../AwsDeallocateStatefulInstanceRequest.java | 10 +- ...ava => AwsStatefulDeAllocationConfig.java} | 2 +- .../aws/stateful/AwsStatefulInstance.java | 22 +- .../model/common/BaseStatefulInstance.java | 26 +++ .../SpotQueueStatefulTaskDispatcher.java | 42 ++++ .../queue/SpotQueueTaskDispatcher.java | 116 ----------- .../spotinst/slave/SpotLauncherHelper.java | 8 +- .../spotinst/slave/SpotinstComputer.java | 8 +- .../plugins/spotinst/slave/SpotinstSlave.java | 107 +++++----- .../plugins/spotinst/SpotinstCloudTest.java | 67 +++--- 23 files changed, 452 insertions(+), 601 deletions(-) delete mode 100644 src/main/java/hudson/plugins/spotinst/common/StatefulInstanceStateEnum.java rename src/main/java/hudson/plugins/spotinst/common/stateful/{StatefulInstanceStateEnum.java => AwsStatefulInstanceStateEnum.java} (77%) create mode 100644 src/main/java/hudson/plugins/spotinst/common/stateful/StatefulInstanceManager.java delete mode 100644 src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulDeallocationConfig.java delete mode 100644 src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstance.java delete mode 100644 src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java delete mode 100644 src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesResponse.java rename src/main/java/hudson/plugins/spotinst/model/aws/stateful/{AwsStatefulDeallocationConfig.java => AwsStatefulDeAllocationConfig.java} (96%) create mode 100644 src/main/java/hudson/plugins/spotinst/model/common/BaseStatefulInstance.java create mode 100644 src/main/java/hudson/plugins/spotinst/queue/SpotQueueStatefulTaskDispatcher.java delete mode 100644 src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java diff --git a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java index c6e36829..e5b4df51 100644 --- a/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java +++ b/src/main/java/hudson/plugins/spotinst/api/SpotinstApi.java @@ -6,7 +6,7 @@ import hudson.plugins.spotinst.common.SpotinstContext; import hudson.plugins.spotinst.model.aws.*; import hudson.plugins.spotinst.model.aws.stateful.AwsDeallocateStatefulInstanceRequest; -import hudson.plugins.spotinst.model.aws.stateful.AwsStatefulDeallocationConfig; +import hudson.plugins.spotinst.model.aws.stateful.AwsStatefulDeAllocationConfig; import hudson.plugins.spotinst.model.aws.stateful.AwsStatefulInstance; import hudson.plugins.spotinst.model.aws.stateful.AwsStatefulInstancesResponse; import hudson.plugins.spotinst.model.azure.*; @@ -177,12 +177,12 @@ public static Boolean awsDeallocateInstance(String groupId, String statefulInsta Map queryParams = buildQueryParams(accountId); AwsDeallocateStatefulInstanceRequest request = new AwsDeallocateStatefulInstanceRequest(); - AwsStatefulDeallocationConfig statefulDeallocation = new AwsStatefulDeallocationConfig(); + AwsStatefulDeAllocationConfig statefulDeallocation = new AwsStatefulDeAllocationConfig(); statefulDeallocation.setShouldDeleteImages(true); statefulDeallocation.setShouldDeleteSnapshots(true); statefulDeallocation.setShouldDeleteVolumes(true); statefulDeallocation.setShouldDeleteNetworkInterfaces(true); - request.setStatefulDeallocation(statefulDeallocation); + request.setStatefulDeAllocationConfig(statefulDeallocation); String body = JsonMapper.toJson(request); diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java index 8a3308c4..f18574f2 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AwsSpotinstCloud.java @@ -6,7 +6,8 @@ import hudson.plugins.spotinst.api.infra.JsonMapper; import hudson.plugins.spotinst.common.ConnectionMethodEnum; import hudson.plugins.spotinst.common.SpotAwsInstanceTypesHelper; -import hudson.plugins.spotinst.common.stateful.StatefulInstanceStateEnum; +import hudson.plugins.spotinst.common.stateful.AwsStatefulInstanceStateEnum; +import hudson.plugins.spotinst.common.stateful.StatefulInstanceManager; import hudson.plugins.spotinst.model.aws.*; import hudson.plugins.spotinst.model.aws.stateful.AwsStatefulInstance; import hudson.plugins.spotinst.model.common.BlResponse; @@ -19,7 +20,6 @@ import jenkins.model.Jenkins; import org.apache.commons.lang.BooleanUtils; import org.apache.commons.lang.StringUtils; -import org.apache.commons.collections.CollectionUtils; import org.kohsuke.stapler.DataBoundConstructor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -107,67 +107,36 @@ List scaleUp(ProvisionRequest request) { return retVal; } - @Override - protected String getStatefulInstanceId(String instanceId) { - String retVal = null; - - IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); - ApiResponse> statefulInstancesResponse = - awsGroupRepo.getStatefulInstances(groupId, accountId); - - if (statefulInstancesResponse.isRequestSucceed()) { - List groupStatefulInstances = statefulInstancesResponse.getValue(); - boolean isGroupHasStatefulInstances = CollectionUtils.isNotEmpty(groupStatefulInstances); - - if (isGroupHasStatefulInstances) { - Optional optionalAwsStatefulInstance = groupStatefulInstances.stream() - .filter(statefulInstance -> instanceId.equals( - statefulInstance.getInstanceId())) - .findFirst(); - boolean isStatefulInstance = optionalAwsStatefulInstance.isPresent(); - - if (isStatefulInstance) { - retVal = optionalAwsStatefulInstance.get().getId(); - } - } - } - - return retVal; - } - - @Override - public Boolean deallocateInstance(String statefulInstanceId) { - boolean retVal = false; - - IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); - ApiResponse detachInstanceResponse = - awsGroupRepo.deallocateInstance(groupId, statefulInstanceId, accountId); - - if (detachInstanceResponse.isRequestSucceed()) { - LOGGER.info(String.format("Stateful Instance %s deallocated", statefulInstanceId)); - retVal = true; - } - else { - LOGGER.error(String.format("Failed to deallocate instance %s. Errors: %s", statefulInstanceId, - detachInstanceResponse.getErrors())); - } - - return retVal; - } + // @Override + // protected String getStatefulInstanceId(String instanceId) { + // String retVal = null; + // + // IAwsGroupRepo awsGroupRepo = RepoManager.getInstance().getAwsGroupRepo(); + // ApiResponse> statefulInstancesResponse = + // awsGroupRepo.getStatefulInstances(groupId, accountId); + // + // if (statefulInstancesResponse.isRequestSucceed()) { + // List groupStatefulInstances = statefulInstancesResponse.getValue(); + // boolean isGroupHasStatefulInstances = CollectionUtils.isNotEmpty(groupStatefulInstances); + // + // if (isGroupHasStatefulInstances) { + // Optional optionalAwsStatefulInstance = groupStatefulInstances.stream() + // .filter(statefulInstance -> instanceId.equals( + // statefulInstance.getInstanceId())) + // .findFirst(); + // boolean isStatefulInstance = optionalAwsStatefulInstance.isPresent(); + // + // if (isStatefulInstance) { + // retVal = optionalAwsStatefulInstance.get().getId(); + // } + // } + // } + // + // return retVal; + // } @Override - protected String getSsiId(String instanceId) { - String retVal = null; - AwsStatefulInstance statefulInstance = getStatefulInstance(instanceId); - - if (statefulInstance != null) { - retVal = statefulInstance.getId(); - } - - return retVal; - } - - private AwsStatefulInstance getStatefulInstance(String instanceId) { + protected AwsStatefulInstance getStatefulInstance(String instanceId) { AwsStatefulInstance retVal = null; boolean isInstanceStateful = ssiByInstanceId != null && ssiByInstanceId.containsKey(instanceId); @@ -357,6 +326,7 @@ private void syncGroupStatefulInstances() { List statefulInstances = statefulInstancesResponse.getValue(); this.ssiByInstanceId = statefulInstances.stream().collect( Collectors.toMap(AwsStatefulInstance::getInstanceId, statefulInstance -> statefulInstance)); + StatefulInstanceManager.getStatefulInstanceIdsByGroupId().put(groupId, statefulInstances); } else { LOGGER.error(String.format("Failed to get group %s stateful instances. Errors: %s", groupId, @@ -371,8 +341,7 @@ private List handleNewAwsSpots(AwsScaleUpResult scaleUpResult, St LOGGER.info(String.format("%s new spot requests created", scaleUpResult.getNewSpotRequests().size())); for (AwsScaleResultNewSpot spot : scaleUpResult.getNewSpotRequests()) { - String ssiId = getSsiByInstance(spot.getInstanceId()); - SpotinstSlave slave = handleNewAwsInstance(spot.getInstanceId(), spot.getInstanceType(), ssiId, label); + SpotinstSlave slave = handleNewAwsInstance(spot.getInstanceId(), spot.getInstanceType(), label); retVal.add(slave); } @@ -386,19 +355,17 @@ private List handleNewAwsInstances(AwsScaleUpResult scaleUpResult LOGGER.info(String.format("%s new instances launched", scaleUpResult.getNewInstances().size())); for (AwsScaleResultNewInstance instance : scaleUpResult.getNewInstances()) { - String ssiId = getSsiByInstance(instance.getInstanceId()); - SpotinstSlave slave = - handleNewAwsInstance(instance.getInstanceId(), instance.getInstanceType(), ssiId, label); + SpotinstSlave slave = handleNewAwsInstance(instance.getInstanceId(), instance.getInstanceType(), label); retVal.add(slave); } return retVal; } - private SpotinstSlave handleNewAwsInstance(String instanceId, String instanceType, String ssiId, String label) { + private SpotinstSlave handleNewAwsInstance(String instanceId, String instanceType, String label) { Integer executors = getNumOfExecutors(instanceType); addToPending(instanceId, executors, PendingInstance.StatusEnum.INSTANCE_INITIATING, label); - SpotinstSlave retVal = buildSpotinstSlave(instanceId, instanceType, ssiId, String.valueOf(executors)); + SpotinstSlave retVal = buildSpotinstSlave(instanceId, instanceType, String.valueOf(executors)); return retVal; } @@ -432,7 +399,7 @@ private Boolean shouldAddNewSlaveInstance(AwsGroupInstance instance) { AwsStatefulInstance statefulInstance = getStatefulInstance(instance.getInstanceId()); boolean isStatefulInstanceReadyForUse = statefulInstance != null && Objects.equals(statefulInstance.getState(), - StatefulInstanceStateEnum.ACTIVE); + AwsStatefulInstanceStateEnum.ACTIVE); retVal = isStatefulInstanceReadyForUse; } else { @@ -532,26 +499,8 @@ private Boolean isSlaveExistForInstance(AwsGroupInstance instance) { return retVal; } - private String getSsiByInstance(String instanceId) { - String retVal = null; - Map ssiById = - AwsStatefulInstancesManager.getAwsStatefulInstanceBySsiByGroupId().get(groupId); - - if(ssiById != null) { - Optional optionalMatchingSsi = - ssiById.values().stream().filter(ssi -> instanceId.equals(ssi.getInstanceId())).findFirst(); - - if (optionalMatchingSsi.isPresent()) { - retVal = optionalMatchingSsi.get().getId(); - } - } - - return retVal; - } - private void addSpotinstSlave(AwsGroupInstance instance) { - String ssiId = getSsiByInstance(instance.getInstanceId()); - SpotinstSlave slave = handleNewAwsInstance(instance.getInstanceId(), instance.getInstanceType(), ssiId, null); + SpotinstSlave slave = handleNewAwsInstance(instance.getInstanceId(), instance.getInstanceType(), null); if (slave != null) { try { diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java index bff19404..fffcbd87 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotCloud.java @@ -9,6 +9,7 @@ import hudson.plugins.spotinst.model.azure.AzureGroupVm; import hudson.plugins.spotinst.model.azure.AzureScaleUpResultNewVm; import hudson.plugins.spotinst.model.azure.AzureVmSizeEnum; +import hudson.plugins.spotinst.model.common.BaseStatefulInstance; import hudson.plugins.spotinst.model.common.BlResponse; import hudson.plugins.spotinst.repos.IAzureVmGroupRepo; import hudson.plugins.spotinst.repos.RepoManager; @@ -41,13 +42,13 @@ public class AzureSpotCloud extends BaseSpotinstCloud { @DataBoundConstructor public AzureSpotCloud(String groupId, String labelString, String idleTerminationMinutes, String workspaceDir, SlaveUsageEnum usage, String tunnel, Boolean shouldUseWebsocket, - Boolean shouldRetriggerBuilds, String vmargs, + SpotReTriggerBuilds spotReTriggerBuilds, String vmargs, EnvironmentVariablesNodeProperty environmentVariables, ToolLocationNodeProperty toolLocations, String accountId, ConnectionMethodEnum connectionMethod, ComputerConnector computerConnector, Boolean shouldUsePrivateIp, SpotGlobalExecutorOverride globalExecutorOverride, Integer pendingThreshold) { super(groupId, labelString, idleTerminationMinutes, workspaceDir, usage, tunnel, shouldUseWebsocket, - shouldRetriggerBuilds, vmargs, environmentVariables, toolLocations, accountId, connectionMethod, + spotReTriggerBuilds, vmargs, environmentVariables, toolLocations, accountId, connectionMethod, computerConnector, shouldUsePrivateIp, globalExecutorOverride, pendingThreshold); } //endregion @@ -92,7 +93,7 @@ protected BlResponse checkIsStatefulGroup() { } @Override - protected String getSsiId(String instanceId) { + protected BaseStatefulInstance getStatefulInstance(String instanceId) { return null;//TODO: implement } diff --git a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java index 600ecf7a..6946feba 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/AzureSpotinstCloud.java @@ -8,6 +8,7 @@ import hudson.plugins.spotinst.common.Constants; import hudson.plugins.spotinst.model.azure.AzureGroupInstance; import hudson.plugins.spotinst.model.azure.AzureScaleSetSizeEnum; +import hudson.plugins.spotinst.model.common.BaseStatefulInstance; import hudson.plugins.spotinst.model.common.BlResponse; import hudson.plugins.spotinst.repos.IAzureGroupRepo; import hudson.plugins.spotinst.repos.RepoManager; @@ -39,14 +40,14 @@ public class AzureSpotinstCloud extends BaseSpotinstCloud { @DataBoundConstructor public AzureSpotinstCloud(String groupId, String labelString, String idleTerminationMinutes, String workspaceDir, SlaveUsageEnum usage, String tunnel, Boolean shouldUseWebsocket, - Boolean shouldRetriggerBuilds, String vmargs, + SpotReTriggerBuilds spotReTriggerBuilds, String vmargs, EnvironmentVariablesNodeProperty environmentVariables, ToolLocationNodeProperty toolLocations, String accountId, ConnectionMethodEnum connectionMethod, ComputerConnector computerConnector, Boolean shouldUsePrivateIp, SpotGlobalExecutorOverride globalExecutorOverride, Integer pendingThreshold) { super(groupId, labelString, idleTerminationMinutes, workspaceDir, usage, tunnel, shouldUseWebsocket, - shouldRetriggerBuilds, vmargs, environmentVariables, toolLocations, accountId, connectionMethod, + spotReTriggerBuilds, vmargs, environmentVariables, toolLocations, accountId, connectionMethod, computerConnector, shouldUsePrivateIp, globalExecutorOverride, pendingThreshold); } //endregion @@ -82,7 +83,7 @@ protected BlResponse checkIsStatefulGroup() { } @Override - protected String getSsiId(String instanceId) { + protected BaseStatefulInstance getStatefulInstance(String instanceId) { return null;//TODO: implement } diff --git a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java index 3a198730..5ddaeb88 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/BaseSpotinstCloud.java @@ -5,6 +5,7 @@ import hudson.model.labels.LabelAtom; import hudson.plugins.spotinst.api.infra.JsonMapper; import hudson.plugins.spotinst.common.*; +import hudson.plugins.spotinst.model.common.BaseStatefulInstance; import hudson.plugins.spotinst.model.common.BlResponse; import hudson.plugins.spotinst.slave.*; import hudson.plugins.sshslaves.SSHConnector; @@ -510,7 +511,7 @@ protected void addToPending(String id, Integer numOfExecutors, PendingInstance.S pendingInstances.put(id, pendingInstance); } - protected SpotinstSlave buildSpotinstSlave(String id, String instanceType, String ssiId, String numOfExecutors) { + protected SpotinstSlave buildSpotinstSlave(String id, String instanceType, String numOfExecutors) { SpotinstSlave slave = null; Node.Mode mode = Node.Mode.NORMAL; @@ -519,6 +520,7 @@ protected SpotinstSlave buildSpotinstSlave(String id, String instanceType, Strin } List> nodeProperties = buildNodeProperties(); + String ssiId = getSsiId(id); try { ComputerLauncher launcher = buildLauncherForAgent(id); @@ -534,10 +536,6 @@ protected SpotinstSlave buildSpotinstSlave(String id, String instanceType, Strin return slave; } - protected SpotinstSlave buildSpotinstSlave(String id, String instanceType, String numOfExecutors) { - return buildSpotinstSlave(id, instanceType, null, numOfExecutors); - } - private ComputerLauncher buildLauncherForAgent(String instanceId) throws IOException { ComputerLauncher retVal; boolean isSshCloud = this.getConnectionMethod().equals(ConnectionMethodEnum.SSH); @@ -763,6 +761,10 @@ public void setShouldUseWebsocket(Boolean shouldUseWebsocket) { } public SpotReTriggerBuilds getSpotReTriggerBuilds() { + if(spotReTriggerBuilds == null){ + spotReTriggerBuilds = new SpotReTriggerBuilds(false, false); + } + return spotReTriggerBuilds; } @@ -771,27 +773,14 @@ public void setSpotReTriggerBuilds(SpotReTriggerBuilds spotReTriggerBuilds) { } public Boolean getShouldReTriggerBuilds() { - Boolean retVal; - - if(getSpotReTriggerBuilds() == null){ - retVal = null; - } - else{ - retVal = getSpotReTriggerBuilds().getShouldReTriggerBuilds(); - } - - return retVal; + return getSpotReTriggerBuilds().getShouldReTriggerBuilds(); } public void setShouldReTriggerBuilds(Boolean shouldReTriggerBuilds) { - this.getSpotReTriggerBuilds().setShouldReTriggerBuilds(shouldReTriggerBuilds); + getSpotReTriggerBuilds().setShouldReTriggerBuilds(shouldReTriggerBuilds); } public Boolean getStickyNode() { - if(getSpotReTriggerBuilds() == null){ - return false; - } - return getSpotReTriggerBuilds().getStickyNode(); } @@ -933,7 +922,18 @@ public Boolean isStatefulGroup() { return isStatefulGroup; } - protected abstract String getSsiId(String instanceId); + public String getSsiId(String instanceId){ + String retVal = null; + BaseStatefulInstance statefulInstance = getStatefulInstance(instanceId); + + if(statefulInstance != null){ + retVal = statefulInstance.getId(); + } + + return retVal; + } + + protected abstract BaseStatefulInstance getStatefulInstance(String instanceId); protected abstract Boolean detachInstance(String instanceId); diff --git a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java index 53184cef..ed4a5886 100644 --- a/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java +++ b/src/main/java/hudson/plugins/spotinst/cloud/GcpSpotinstCloud.java @@ -5,6 +5,7 @@ import hudson.plugins.spotinst.api.infra.ApiResponse; import hudson.plugins.spotinst.api.infra.JsonMapper; import hudson.plugins.spotinst.common.ConnectionMethodEnum; +import hudson.plugins.spotinst.model.common.BaseStatefulInstance; import hudson.plugins.spotinst.model.common.BlResponse; import hudson.plugins.spotinst.model.gcp.GcpGroupInstance; import hudson.plugins.spotinst.model.gcp.GcpMachineType; @@ -43,14 +44,14 @@ public class GcpSpotinstCloud extends BaseSpotinstCloud { @DataBoundConstructor public GcpSpotinstCloud(String groupId, String labelString, String idleTerminationMinutes, String workspaceDir, SlaveUsageEnum usage, String tunnel, Boolean shouldUseWebsocket, - Boolean shouldRetriggerBuilds, String vmargs, + SpotReTriggerBuilds spotReTriggerBuilds, String vmargs, EnvironmentVariablesNodeProperty environmentVariables, ToolLocationNodeProperty toolLocations, String accountId, ConnectionMethodEnum connectionMethod, ComputerConnector computerConnector, Boolean shouldUsePrivateIp, SpotGlobalExecutorOverride globalExecutorOverride, Integer pendingThreshold) { super(groupId, labelString, idleTerminationMinutes, workspaceDir, usage, tunnel, shouldUseWebsocket, - shouldRetriggerBuilds, vmargs, environmentVariables, toolLocations, accountId, connectionMethod, + spotReTriggerBuilds, vmargs, environmentVariables, toolLocations, accountId, connectionMethod, computerConnector, shouldUsePrivateIp, globalExecutorOverride, pendingThreshold); } //endregion @@ -105,7 +106,7 @@ protected BlResponse checkIsStatefulGroup() { } @Override - protected String getSsiId(String instanceId) { + protected BaseStatefulInstance getStatefulInstance(String instanceId) { return null;//TODO: implement } diff --git a/src/main/java/hudson/plugins/spotinst/common/StatefulInstanceStateEnum.java b/src/main/java/hudson/plugins/spotinst/common/StatefulInstanceStateEnum.java deleted file mode 100644 index 6ac57229..00000000 --- a/src/main/java/hudson/plugins/spotinst/common/StatefulInstanceStateEnum.java +++ /dev/null @@ -1,62 +0,0 @@ -package hudson.plugins.spotinst.common; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Created by sitay on 30/08/23. - */ -public enum StatefulInstanceStateEnum { - //region Enums - ACTIVE("ACTIVE"), - PAUSE("PAUSE"), - PAUSING("PAUSING"), - PAUSED("PAUSED"), - RESUME("RESUME"), - RESUMING("RESUMING"), - RECYCLE("RECYCLE"), - RECYCLING("RECYCLING"), - DEALLOCATE("DEALLOCATE"), - DEALLOCATING("DEALLOCATING"), - DEALLOCATED("DEALLOCATED"), - ERROR("ERROR"); - //endregion - - //region Members - private static final Logger LOGGER = LoggerFactory.getLogger(StatefulInstanceStateEnum.class); - private final String name; - //endregion - - //region Constructors - StatefulInstanceStateEnum(String name) { - this.name = name; - } - //endregion - - //region Getters & Setters - public String getName() { - return name; - } - //endregion - - //region Methods - public static StatefulInstanceStateEnum fromName(String name) { - StatefulInstanceStateEnum retVal = null; - - for (StatefulInstanceStateEnum deploymentInstanceStatus : StatefulInstanceStateEnum.values()) { - if (name.equalsIgnoreCase(deploymentInstanceStatus.name)) { - retVal = deploymentInstanceStatus; - break; - } - } - - if (retVal == null) { - LOGGER.error( - "Tried to create stateful instance state Enum for: " + name + ", but we don't support such type "); - } - - return retVal; - } - //endregion -} - diff --git a/src/main/java/hudson/plugins/spotinst/common/stateful/StatefulInstanceStateEnum.java b/src/main/java/hudson/plugins/spotinst/common/stateful/AwsStatefulInstanceStateEnum.java similarity index 77% rename from src/main/java/hudson/plugins/spotinst/common/stateful/StatefulInstanceStateEnum.java rename to src/main/java/hudson/plugins/spotinst/common/stateful/AwsStatefulInstanceStateEnum.java index 0479bde3..8473c8d3 100644 --- a/src/main/java/hudson/plugins/spotinst/common/stateful/StatefulInstanceStateEnum.java +++ b/src/main/java/hudson/plugins/spotinst/common/stateful/AwsStatefulInstanceStateEnum.java @@ -6,7 +6,7 @@ /** * Created by sitay on 30/08/23. */ -public enum StatefulInstanceStateEnum { +public enum AwsStatefulInstanceStateEnum { //region Enums ACTIVE("ACTIVE"), PAUSE("PAUSE"), @@ -23,12 +23,12 @@ public enum StatefulInstanceStateEnum { //endregion //region Members - private static final Logger LOGGER = LoggerFactory.getLogger(StatefulInstanceStateEnum.class); + private static final Logger LOGGER = LoggerFactory.getLogger(AwsStatefulInstanceStateEnum.class); private final String name; //endregion //region Constructors - StatefulInstanceStateEnum(String name) { + AwsStatefulInstanceStateEnum(String name) { this.name = name; } //endregion @@ -40,10 +40,10 @@ public String getName() { //endregion //region Methods - public static StatefulInstanceStateEnum fromName(String name) { - StatefulInstanceStateEnum retVal = null; + public static AwsStatefulInstanceStateEnum fromName(String name) { + AwsStatefulInstanceStateEnum retVal = null; - for (StatefulInstanceStateEnum deploymentInstanceStatus : StatefulInstanceStateEnum.values()) { + for (AwsStatefulInstanceStateEnum deploymentInstanceStatus : AwsStatefulInstanceStateEnum.values()) { if (name.equalsIgnoreCase(deploymentInstanceStatus.name)) { retVal = deploymentInstanceStatus; break; diff --git a/src/main/java/hudson/plugins/spotinst/common/stateful/StatefulInstanceManager.java b/src/main/java/hudson/plugins/spotinst/common/stateful/StatefulInstanceManager.java new file mode 100644 index 00000000..c47ce3db --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/common/stateful/StatefulInstanceManager.java @@ -0,0 +1,195 @@ +package hudson.plugins.spotinst.common.stateful; + +import hudson.model.Executor; +import hudson.model.Node; +import hudson.model.Queue; +import hudson.model.queue.CauseOfBlockage; +import hudson.plugins.spotinst.model.aws.stateful.AwsStatefulInstance; +import hudson.plugins.spotinst.model.common.BaseStatefulInstance; +import hudson.plugins.spotinst.slave.SpotinstSlave; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; + +public class StatefulInstanceManager { + //region members + private static final Logger LOGGER = + LoggerFactory.getLogger(StatefulInstanceManager.class); + private static final Map> statefulInstanceIdsByGroupId = + new ConcurrentHashMap<>(); + private static final Map ssiByStatefulTask = + new ConcurrentHashMap<>(); + private static final Set statefulTasksToReTrigger = + new HashSet<>(); + //endregion + + //region methods + public static CauseOfBlockage canNodeTakeStatefulTask(Node node, String statefulTaskSsi) { + CauseOfBlockage retVal = null; + Optional> optionalMatchingGroupStatefulInstances = + statefulInstanceIdsByGroupId.values().stream().filter(statefulInstances -> statefulInstances.stream() + .anyMatch( + statefulInstance -> statefulInstance.getId() + .equals(statefulTaskSsi))) + .findFirst(); + boolean isStatefulInstanceExist = optionalMatchingGroupStatefulInstances.isPresent(); + + if (isStatefulInstanceExist) { + + Optional optionalMatchingStatefulInstance = + optionalMatchingGroupStatefulInstances.get().stream() + .filter(statefulInstance -> statefulInstance.getId() + .equals(statefulTaskSsi)) + .findFirst(); + + if (optionalMatchingStatefulInstance.isPresent()) { + retVal = canStatefulInstanceTakeStatefulTask(node, optionalMatchingStatefulInstance.get(), + statefulTaskSsi); + } + } + else { + LOGGER.info( + "SSI '{}' does not exist any more. reserved tasks for the stateful instance can be taken by any node", + statefulTaskSsi); + } + + return retVal; + } + + public static void putSsiByTask(Queue.Task task, Executor executor, String ssiId) { + String key = generateKey(task, executor); + ssiByStatefulTask.put(key, ssiId); + } + + public static String removeSsiByTask(Queue.Task task, Executor executor) { + String retVal; + String key = generateKey(task, executor); + retVal = ssiByStatefulTask.remove(key); + return retVal; + } + + + public static Boolean isReTriggeringStatefulTask(Queue.Task task, Executor executor) { + boolean retVal; + String key = generateKey(task, executor); + retVal = statefulTasksToReTrigger.contains(key); + return retVal; + } + + public static void handleReTriggeringStatefulTask(Queue.Task task, Executor executor) { + String key = generateKey(task, executor); + statefulTasksToReTrigger.add(key); + } + + public static void handleReTriggeredStatefulTask(Queue.Task task, Executor executor) { + String key = generateKey(task, executor); + statefulTasksToReTrigger.remove(key); + } + //endregion + + //region private methods + private static String generateKey(Queue.Task task, Executor executor) { + return String.format("%s_%s", task, executor.getId()); + } + + private static CauseOfBlockage canStatefulInstanceTakeStatefulTask(Node node, BaseStatefulInstance statefulInstance, + String statefulTaskSsi) { + CauseOfBlockage retVal; + + if (statefulInstance instanceof AwsStatefulInstance) { + retVal = canAwsStatefulInstanceTakeStatefulTask(node, (AwsStatefulInstance) statefulInstance, + statefulTaskSsi); + } + else { + LOGGER.warn("stateful instance of unsupported cloud '{}'", statefulTaskSsi); + retVal = new InconsistentSsiCauseOfBlockage(); + } + + return retVal; + } + + private static CauseOfBlockage canAwsStatefulInstanceTakeStatefulTask(Node node, + AwsStatefulInstance statefulInstance, + String statefulTaskSsi) { + CauseOfBlockage retVal = null; + AwsStatefulInstanceStateEnum ssiState = statefulInstance.getState(); + + switch (ssiState) { + case ACTIVE: + boolean isNodeMatchingTaskSsi = checkMatchingSsis(node, statefulTaskSsi); + + if (isNodeMatchingTaskSsi) { + LOGGER.info("found slave for stateful task with SSI '{}'", statefulTaskSsi); + } + else { + retVal = new InconsistentSsiCauseOfBlockage(); + } + + break; + + case RECYCLE: + case RECYCLING: + case RESUME: + case RESUMING: + LOGGER.info("stateful task awaits for SSI {} to be active. no node can take it", statefulTaskSsi); + retVal = new AwaitingSsiCauseOfBlockage(); + break; + + case PAUSE: + case PAUSING: + case PAUSED: + case DEALLOCATE: + case DEALLOCATING: + case DEALLOCATED: + case ERROR: + LOGGER.info("SSI {} is in state {} and isn't activated, stateful task can run on any node", + statefulTaskSsi, ssiState); + break; + } + + return retVal; + } + + private static Boolean checkMatchingSsis(Node node, String statefulTaskSsi) { + boolean retVal = false; + + if (node instanceof SpotinstSlave) { + SpotinstSlave slave = (SpotinstSlave) node; + String ssi = slave.getSsiId(); + boolean isStatefulSlave = StringUtils.isNotEmpty(ssi); + + if (isStatefulSlave) { + retVal = ssi.equals(statefulTaskSsi); + } + } + + return retVal; + } + + //endregion + + //region getters & setters + public static Map> getStatefulInstanceIdsByGroupId() { + return statefulInstanceIdsByGroupId; + } + //endregion + + //region classes + private static class InconsistentSsiCauseOfBlockage extends CauseOfBlockage { + @Override + public String getShortDescription() { + return "task reserved for slave with ssi different from current node"; + } + } + + private static class AwaitingSsiCauseOfBlockage extends CauseOfBlockage { + @Override + public String getShortDescription() { + return "awaiting for stateful node to recover"; + } + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulDeallocationConfig.java b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulDeallocationConfig.java deleted file mode 100644 index 507fa01b..00000000 --- a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulDeallocationConfig.java +++ /dev/null @@ -1,50 +0,0 @@ -package hudson.plugins.spotinst.model.aws; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -/** - * Created by sitay on 30/08/23. - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class AwsStatefulDeallocationConfig { - //region members - private Boolean shouldDeleteImages; - private Boolean shouldDeleteNetworkInterfaces; - private Boolean shouldDeleteVolumes; - private Boolean shouldDeleteSnapshots; - //endregion - - //region getters & setters - public Boolean getShouldDeleteImages() { - return shouldDeleteImages; - } - - public void setShouldDeleteImages(Boolean shouldDeleteImages) { - this.shouldDeleteImages = shouldDeleteImages; - } - - public Boolean getShouldDeleteNetworkInterfaces() { - return shouldDeleteNetworkInterfaces; - } - - public void setShouldDeleteNetworkInterfaces(Boolean shouldDeleteNetworkInterfaces) { - this.shouldDeleteNetworkInterfaces = shouldDeleteNetworkInterfaces; - } - - public Boolean getShouldDeleteVolumes() { - return shouldDeleteVolumes; - } - - public void setShouldDeleteVolumes(Boolean shouldDeleteVolumes) { - this.shouldDeleteVolumes = shouldDeleteVolumes; - } - - public Boolean getShouldDeleteSnapshots() { - return shouldDeleteSnapshots; - } - - public void setShouldDeleteSnapshots(Boolean shouldDeleteSnapshots) { - this.shouldDeleteSnapshots = shouldDeleteSnapshots; - } - //endregion -} diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstance.java b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstance.java deleted file mode 100644 index 8836e08f..00000000 --- a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstance.java +++ /dev/null @@ -1,44 +0,0 @@ -package hudson.plugins.spotinst.model.aws; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import hudson.plugins.spotinst.common.StatefulInstanceStateEnum; - -/** - * Created by ItayShklar on 07/08/2023. - */ -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -public class AwsStatefulInstance { - //region members - private String id; - private String instanceId; - private StatefulInstanceStateEnum state; - //endregion - - //region getters & setters - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public String getInstanceId() { - return instanceId; - } - - public void setInstanceId(String instanceId) { - this.instanceId = instanceId; - } - - public StatefulInstanceStateEnum getState() { - return state; - } - - public void setState(StatefulInstanceStateEnum state) { - this.state = state; - } - //endregion -} diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java deleted file mode 100644 index 191feb8d..00000000 --- a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesManager.java +++ /dev/null @@ -1,82 +0,0 @@ -package hudson.plugins.spotinst.model.aws; - -import hudson.model.Executor; -import hudson.model.Queue; - -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; - -/** - * Created by sitay on 30/08/2013. - */ -public class AwsStatefulInstancesManager { - //region members - private static final Map ssiByStatefulTask = - new ConcurrentHashMap<>(); - private static final Map> awsStatefulInstanceBySsiByGroupId = - new ConcurrentHashMap<>(); - private static final Set statefulTasksToReTrigger = - new HashSet<>(); - //endregion - - //region methods - public static void putSsiByTask(Queue.Task task, Executor executor, String ssiId) { - String key = generateKey(task, executor); - ssiByStatefulTask.put(key, ssiId); - } - - public static String removeSsiByTask(Queue.Task task, Executor executor) { - String retVal; - String key = generateKey(task, executor); - retVal = ssiByStatefulTask.remove(key); - return retVal; - } - - public static AwsStatefulInstance getStatefulInstanceBySSi(String ssi) { - AwsStatefulInstance retVal = null; - Collection> allGroupsSsiById = awsStatefulInstanceBySsiByGroupId.values(); - - Optional optionalMatchingStatefulInstance = - allGroupsSsiById.stream().filter(groupSsiById -> groupSsiById.containsKey(ssi)) - .map(groupSsiById -> groupSsiById.get(ssi)).findFirst(); - - if (optionalMatchingStatefulInstance.isPresent()) { - retVal = optionalMatchingStatefulInstance.get(); - } - - return retVal; - } - - - public static Boolean isReTriggeringStatefulTask(Queue.Task task, Executor executor) { - boolean retVal; - String key = generateKey(task, executor); - retVal = statefulTasksToReTrigger.contains(key); - return retVal; - } - - public static void handleReTriggeringStatefulTask(Queue.Task task, Executor executor) { - String key = generateKey(task, executor); - statefulTasksToReTrigger.add(key); - } - - public static Boolean handleReTriggeredStatefulTask(Queue.Task task, Executor executor) { - boolean retVal; - String key = generateKey(task, executor); - retVal = statefulTasksToReTrigger.remove(key); - return retVal; - } - //endregion - - //region private methods - private static String generateKey(Queue.Task task, Executor executor) { - return String.format("%s_%s", task, executor.getId()); - } - //endregion - - //region getters & setters - public static Map> getAwsStatefulInstanceBySsiByGroupId() { - return awsStatefulInstanceBySsiByGroupId; - } - //endregion -} diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesResponse.java b/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesResponse.java deleted file mode 100644 index af564037..00000000 --- a/src/main/java/hudson/plugins/spotinst/model/aws/AwsStatefulInstancesResponse.java +++ /dev/null @@ -1,6 +0,0 @@ -package hudson.plugins.spotinst.model.aws; - -import hudson.plugins.spotinst.api.infra.BaseItemsResponse; - -public class AwsStatefulInstancesResponse extends BaseItemsResponse { -} diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/stateful/AwsDeallocateStatefulInstanceRequest.java b/src/main/java/hudson/plugins/spotinst/model/aws/stateful/AwsDeallocateStatefulInstanceRequest.java index fad0a7c3..60f9e927 100644 --- a/src/main/java/hudson/plugins/spotinst/model/aws/stateful/AwsDeallocateStatefulInstanceRequest.java +++ b/src/main/java/hudson/plugins/spotinst/model/aws/stateful/AwsDeallocateStatefulInstanceRequest.java @@ -8,16 +8,16 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class AwsDeallocateStatefulInstanceRequest { //region members - private AwsStatefulDeallocationConfig statefulDeallocation; + private AwsStatefulDeAllocationConfig statefulDeAllocationConfig; //endregion //region getters & setters - public AwsStatefulDeallocationConfig getStatefulDeallocation() { - return statefulDeallocation; + public AwsStatefulDeAllocationConfig getStatefulDeAllocationConfig() { + return statefulDeAllocationConfig; } - public void setStatefulDeallocation(AwsStatefulDeallocationConfig statefulDeallocation) { - this.statefulDeallocation = statefulDeallocation; + public void setStatefulDeAllocationConfig(AwsStatefulDeAllocationConfig statefulDeAllocationConfig) { + this.statefulDeAllocationConfig = statefulDeAllocationConfig; } //endregion } \ No newline at end of file diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/stateful/AwsStatefulDeallocationConfig.java b/src/main/java/hudson/plugins/spotinst/model/aws/stateful/AwsStatefulDeAllocationConfig.java similarity index 96% rename from src/main/java/hudson/plugins/spotinst/model/aws/stateful/AwsStatefulDeallocationConfig.java rename to src/main/java/hudson/plugins/spotinst/model/aws/stateful/AwsStatefulDeAllocationConfig.java index 7cb46075..67cf20ba 100644 --- a/src/main/java/hudson/plugins/spotinst/model/aws/stateful/AwsStatefulDeallocationConfig.java +++ b/src/main/java/hudson/plugins/spotinst/model/aws/stateful/AwsStatefulDeAllocationConfig.java @@ -6,7 +6,7 @@ * Created by sitay on 30/08/23. */ @JsonIgnoreProperties(ignoreUnknown = true) -public class AwsStatefulDeallocationConfig { +public class AwsStatefulDeAllocationConfig { //region members private Boolean shouldDeleteImages; private Boolean shouldDeleteNetworkInterfaces; diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/stateful/AwsStatefulInstance.java b/src/main/java/hudson/plugins/spotinst/model/aws/stateful/AwsStatefulInstance.java index e02bbea1..20b4e560 100644 --- a/src/main/java/hudson/plugins/spotinst/model/aws/stateful/AwsStatefulInstance.java +++ b/src/main/java/hudson/plugins/spotinst/model/aws/stateful/AwsStatefulInstance.java @@ -2,29 +2,21 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; -import hudson.plugins.spotinst.common.stateful.StatefulInstanceStateEnum; +import hudson.plugins.spotinst.common.stateful.AwsStatefulInstanceStateEnum; +import hudson.plugins.spotinst.model.common.BaseStatefulInstance; /** * Created by ItayShklar on 07/08/2023. */ @JsonIgnoreProperties(ignoreUnknown = true) @JsonInclude(JsonInclude.Include.NON_NULL) -public class AwsStatefulInstance { +public class AwsStatefulInstance extends BaseStatefulInstance { //region members - private String id; - private String instanceId; - private StatefulInstanceStateEnum state; + private String instanceId; + private AwsStatefulInstanceStateEnum state; //endregion //region getters & setters - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - public String getInstanceId() { return instanceId; } @@ -33,11 +25,11 @@ public void setInstanceId(String instanceId) { this.instanceId = instanceId; } - public StatefulInstanceStateEnum getState() { + public AwsStatefulInstanceStateEnum getState() { return state; } - public void setState(StatefulInstanceStateEnum state) { + public void setState(AwsStatefulInstanceStateEnum state) { this.state = state; } //endregion diff --git a/src/main/java/hudson/plugins/spotinst/model/common/BaseStatefulInstance.java b/src/main/java/hudson/plugins/spotinst/model/common/BaseStatefulInstance.java new file mode 100644 index 00000000..10f19b25 --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/model/common/BaseStatefulInstance.java @@ -0,0 +1,26 @@ +package hudson.plugins.spotinst.model.common; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; + +/** + * Created by ItayShklar on 07/08/2023. + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class BaseStatefulInstance { + //region members + private String id; + //endregion + + //region getters & setters + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + //endregion +} + diff --git a/src/main/java/hudson/plugins/spotinst/queue/SpotQueueStatefulTaskDispatcher.java b/src/main/java/hudson/plugins/spotinst/queue/SpotQueueStatefulTaskDispatcher.java new file mode 100644 index 00000000..189fa1ee --- /dev/null +++ b/src/main/java/hudson/plugins/spotinst/queue/SpotQueueStatefulTaskDispatcher.java @@ -0,0 +1,42 @@ +package hudson.plugins.spotinst.queue; + +import hudson.Extension; +import hudson.model.Node; +import hudson.model.Queue; +import hudson.model.queue.CauseOfBlockage; +import hudson.model.queue.QueueTaskDispatcher; +import hudson.plugins.spotinst.common.stateful.StatefulInstanceManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.CheckForNull; + + +/** + * Created by sitay on 30/08/2013. + */ +@Extension +public class SpotQueueStatefulTaskDispatcher extends QueueTaskDispatcher { + //region members + private static final Logger LOGGER = LoggerFactory.getLogger(SpotQueueStatefulTaskDispatcher.class); + //endregion + + //region override methods + @Override + @CheckForNull + public CauseOfBlockage canTake(Node node, Queue.BuildableItem item) { + CauseOfBlockage retVal = null; + LOGGER.info("can node {} take item {}", this, item);//TODO: remove + Queue.Task task = item.task; + boolean isTaskReservedForStatefulSlave = task instanceof StatefulInterruptedTask; + + if (isTaskReservedForStatefulSlave) { + StatefulInterruptedTask statefulInterruptedTask = (StatefulInterruptedTask) task; + String statefulTaskSsi = statefulInterruptedTask.getSsi(); + retVal = StatefulInstanceManager.canNodeTakeStatefulTask(node, statefulTaskSsi); + } + + return retVal; + } + //endregion +} diff --git a/src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java b/src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java deleted file mode 100644 index 959bf42a..00000000 --- a/src/main/java/hudson/plugins/spotinst/queue/SpotQueueTaskDispatcher.java +++ /dev/null @@ -1,116 +0,0 @@ -//package hudson.plugins.spotinst.queue; -// -//import hudson.Extension; -//import hudson.model.Node; -//import hudson.model.Queue; -//import hudson.model.queue.CauseOfBlockage; -//import hudson.model.queue.QueueTaskDispatcher; -//import hudson.plugins.spotinst.model.aws.AwsStatefulInstance; -//import hudson.plugins.spotinst.model.aws.AwsStatefulInstancesManager; -//import hudson.plugins.spotinst.slave.SpotinstSlave; -//import org.apache.commons.lang.StringUtils; -//import org.slf4j.Logger; -//import org.slf4j.LoggerFactory; -// -// -///** -// * Created by sitay on 30/08/2013. -// */ -//@Extension -//public class SpotQueueTaskDispatcher extends QueueTaskDispatcher { -// //region members -// private static final Logger LOGGER = LoggerFactory.getLogger(SpotQueueTaskDispatcher.class); -// //endregion -// -// //region override methods -// @Override -// public CauseOfBlockage canTake(Node node, Queue.Task task) { -// CauseOfBlockage retVal = null; -// boolean isTaskReservedForStatefulSlave = task instanceof StatefulInterruptedTask; -// -// if (isTaskReservedForStatefulSlave) { -// StatefulInterruptedTask statefulInterruptedTask = (StatefulInterruptedTask) task; -// String statefulTaskSsi = statefulInterruptedTask.getSsi(); -// AwsStatefulInstance matchingStatefulInstance = -// AwsStatefulInstancesManager.getStatefulInstanceBySSi(statefulTaskSsi); -// boolean isSsiExist = matchingStatefulInstance != null; -// -// if (isSsiExist) { -// switch (matchingStatefulInstance.getState()) { -// case PAUSE: -// case PAUSING: -// case PAUSED: -// case DEALLOCATE: -// case DEALLOCATING: -// case DEALLOCATED: -// case ERROR: -// //SSI isn't activated, can run on any node -// break; -// -// case ACTIVE: -// //can only run on specific SSI -// retVal = checkMatchingSsis(node, statefulTaskSsi); -// break; -// -// case RECYCLE: -// case RECYCLING: -// case RESUME: -// case RESUMING: -// default: -// //await for SSI to be active -// retVal = new AwaitingSsiCauseOfBlockage(); -// break; -// } -// } -// } -// -// return retVal; -// } -// //endregion -// -// //region private methods -// private CauseOfBlockage checkMatchingSsis(Node node, String statefulTaskSsi) { -// CauseOfBlockage retVal = null; -// -// if (node instanceof SpotinstSlave) { -// SpotinstSlave slave = (SpotinstSlave) node; -// String ssi = slave.getSsiId(); -// boolean isStatefulSlave = StringUtils.isNotEmpty(ssi); -// -// if (isStatefulSlave) { -// boolean canNodeTakeTask = ssi.equals(statefulTaskSsi); -// -// if (canNodeTakeTask) { -// LOGGER.info("node {} with ssi {} can take task", slave.getNodeName(), ssi); -// } -// else { -// retVal = new InconsistentSsiCauseOfBlockage(); -// } -// } -// else { -// retVal = new InconsistentSsiCauseOfBlockage(); -// } -// } -// else { -// retVal = new InconsistentSsiCauseOfBlockage(); -// } -// -// return retVal; -// } -// -// //region classes -// private static class InconsistentSsiCauseOfBlockage extends CauseOfBlockage { -// @Override -// public String getShortDescription() { -// return "task reserved for slave with ssi different from current node"; -// } -// } -// -// private static class AwaitingSsiCauseOfBlockage extends CauseOfBlockage { -// @Override -// public String getShortDescription() { -// return "awaiting for stateful node to recover"; -// } -// } -// //endregion -//} diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java b/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java index 2f694e73..78950867 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java @@ -3,7 +3,7 @@ import hudson.model.*; import hudson.model.Queue; import hudson.model.queue.SubTask; -import hudson.plugins.spotinst.model.aws.AwsStatefulInstancesManager; +import hudson.plugins.spotinst.common.stateful.StatefulInstanceManager; import hudson.plugins.spotinst.queue.StatefulInterruptedTask; import hudson.slaves.SlaveComputer; import org.apache.commons.lang.BooleanUtils; @@ -56,10 +56,10 @@ static void handleDisconnect(final SlaveComputer computer, Boolean shouldRetrigg synchronized (lock) { - String ssiByTaskName = AwsStatefulInstancesManager.removeSsiByTask(task, executor); + String ssiByTaskName = StatefulInstanceManager.removeSsiByTask(task, executor); if (ssiByTaskName != null) { - AwsStatefulInstancesManager.handleReTriggeringStatefulTask(task, executor); + StatefulInstanceManager.handleReTriggeringStatefulTask(task, executor); StatefulInterruptedTask statefulInterruptedTask = new StatefulInterruptedTask(ssiByTaskName, task); LOGGER.info(String.format("RETRIGGERING Stateful Task: %s - WITH ACTIONS: %s on SSI %s", @@ -68,7 +68,7 @@ static void handleDisconnect(final SlaveComputer computer, Boolean shouldRetrigg Queue.getInstance().schedule2(statefulInterruptedTask, 10, actions); } else { - if (AwsStatefulInstancesManager.isReTriggeringStatefulTask(task, executor)) { + if (StatefulInstanceManager.isReTriggeringStatefulTask(task, executor)) { LOGGER.info(String.format( "Stateful Task : %s is already queued for re-triggering - ignoring it", task)); diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java index 21aabe33..112908fc 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java @@ -4,7 +4,7 @@ import hudson.model.Node; import hudson.model.Queue; import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; -import hudson.plugins.spotinst.model.aws.AwsStatefulInstancesManager; +import hudson.plugins.spotinst.common.stateful.StatefulInstanceManager; import hudson.slaves.SlaveComputer; import org.apache.commons.lang.StringUtils; import org.kohsuke.stapler.HttpRedirect; @@ -54,7 +54,7 @@ public void taskAccepted(Executor executor, Queue.Task task) { if (StringUtils.isNotEmpty(spotinstNodeSsiId)) { LOGGER.info("task {} accepted on executor {} and is bound to ssi {}", task, executor.getId(), spotinstNodeSsiId); - AwsStatefulInstancesManager.putSsiByTask(task, executor, spotinstNodeSsiId); + StatefulInstanceManager.putSsiByTask(task, executor, spotinstNodeSsiId); } else { LOGGER.info("task {} accepted on a stateless slave {}. no restrictions occur", task.getName(), @@ -91,11 +91,11 @@ public void taskCompleted(Executor executor, Queue.Task task, long durationMS) { if (isStatefulNode) { LOGGER.info("task {} accepted on executor {} and is bound to ssi {}", task, executor.getId(), spotinstNodeSsiId); - String attachedSsi = AwsStatefulInstancesManager.removeSsiByTask(task, executor); + String attachedSsi = StatefulInstanceManager.removeSsiByTask(task, executor); boolean isTaskReTriggered = StringUtils.isEmpty(attachedSsi); if (isTaskReTriggered) { - AwsStatefulInstancesManager.handleReTriggeredStatefulTask(task, executor); + StatefulInstanceManager.handleReTriggeredStatefulTask(task, executor); } else { LOGGER.info("unbinding stateful task {} and executor {} from ssi {}", task.getName(), diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java index 1539d00f..bb4afbb9 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstSlave.java @@ -4,10 +4,6 @@ import hudson.model.*; import hudson.model.queue.CauseOfBlockage; import hudson.plugins.spotinst.cloud.BaseSpotinstCloud; -import hudson.plugins.spotinst.common.StatefulInstanceStateEnum; -import hudson.plugins.spotinst.model.aws.AwsStatefulInstance; -import hudson.plugins.spotinst.model.aws.AwsStatefulInstancesManager; -import hudson.plugins.spotinst.queue.StatefulInterruptedTask; import hudson.slaves.*; import jenkins.model.Jenkins; import net.sf.json.JSONObject; @@ -187,56 +183,59 @@ public Node asNode() { return this; } - @Override - public CauseOfBlockage canTake(Queue.BuildableItem item){ - CauseOfBlockage retVal = super.canTake(item); - - if(retVal == null) { - LOGGER.info("can node {} take item {}", this, item);//TODO: remove - Queue.Task task = item.task; - boolean isTaskReservedForStatefulSlave = task instanceof StatefulInterruptedTask; - - if (isTaskReservedForStatefulSlave) { - StatefulInterruptedTask statefulInterruptedTask = (StatefulInterruptedTask) task; - String statefulTaskSsi = statefulInterruptedTask.getSsi(); - AwsStatefulInstance matchingStatefulInstance = AwsStatefulInstancesManager.getStatefulInstanceBySSi(statefulTaskSsi); - boolean isSsiExist = matchingStatefulInstance != null; - - if (isSsiExist) { - StatefulInstanceStateEnum ssiState = matchingStatefulInstance.getState(); - - switch (ssiState) { - case PAUSE: - case PAUSING: - case PAUSED: - case DEALLOCATE: - case DEALLOCATING: - case DEALLOCATED: - case ERROR: - LOGGER.info("SSI {} is in state {} and isn't activated, task {} can run on any node", - statefulTaskSsi, ssiState, task.getName()); - break; - - case ACTIVE: - retVal = checkMatchingSsis(statefulTaskSsi); - break; - - case RECYCLE: - case RECYCLING: - case RESUME: - case RESUMING: - default: - LOGGER.info("task {} awaits for SSI {} to be active. no node can take it", task.getName(), - statefulTaskSsi); - retVal = new AwaitingSsiCauseOfBlockage(); - break; - } - } - } - } - - return retVal; - } +// @Override +// public CauseOfBlockage canTake(Queue.BuildableItem item){ +// CauseOfBlockage retVal = super.canTake(item); +// +// if(retVal == null) { +// LOGGER.info("can node {} take item {}", this, item);//TODO: remove +// Queue.Task task = item.task; +// boolean isTaskReservedForStatefulSlave = task instanceof StatefulInterruptedTask; +// +// if (isTaskReservedForStatefulSlave) { +// StatefulInterruptedTask statefulInterruptedTask = (StatefulInterruptedTask) task; +// String statefulTaskSsi = statefulInterruptedTask.getSsi(); +// BaseSpotinstCloud slaveSpotinstCloud = this.getSpotinstCloud(); +// retVal = slaveSpotinstCloud.canTakeStatefulTask(this.getInstanceId(), statefulTaskSsi); +// AwsStatefulInstance +// matchingStatefulInstance = AwsStatefulInstancesManager.getStatefulInstanceBySSi(statefulTaskSsi); +// boolean isSsiExist = matchingStatefulInstance != null; +// +// if (isSsiExist) { +// AwsStatefulInstanceStateEnum ssiState = matchingStatefulInstance.getState(); +// +// switch (ssiState) { +// case PAUSE: +// case PAUSING: +// case PAUSED: +// case DEALLOCATE: +// case DEALLOCATING: +// case DEALLOCATED: +// case ERROR: +// LOGGER.info("SSI {} is in state {} and isn't activated, task {} can run on any node", +// statefulTaskSsi, ssiState, task.getName()); +// break; +// +// case ACTIVE: +// retVal = checkMatchingSsis(statefulTaskSsi); +// break; +// +// case RECYCLE: +// case RECYCLING: +// case RESUME: +// case RESUMING: +// default: +// LOGGER.info("task {} awaits for SSI {} to be active. no node can take it", task.getName(), +// statefulTaskSsi); +// retVal = new AwaitingSsiCauseOfBlockage(); +// break; +// } +// } +// } +// } +// +// return retVal; +// } private CauseOfBlockage checkMatchingSsis(String statefulTaskSsi) { CauseOfBlockage retVal = null; diff --git a/src/test/java/hudson/plugins/spotinst/SpotinstCloudTest.java b/src/test/java/hudson/plugins/spotinst/SpotinstCloudTest.java index d08a6816..70d26bbc 100644 --- a/src/test/java/hudson/plugins/spotinst/SpotinstCloudTest.java +++ b/src/test/java/hudson/plugins/spotinst/SpotinstCloudTest.java @@ -42,6 +42,9 @@ public void setUp() { IAzureGroupRepo azureGroupRepo = Mockito.mock(IAzureGroupRepo.class); IAzureVmGroupRepo azureVmGroupRepo = Mockito.mock(IAzureVmGroupRepo.class); + Mockito.when(awsGroupRepo.getGroup(Mockito.anyString(), Mockito.anyString())) + .thenReturn(new ApiResponse<>(false)); + RepoManager.getInstance().setAwsGroupRepo(awsGroupRepo); RepoManager.getInstance().setGcpGroupRepo(gcpGroupRepo); RepoManager.getInstance().setAzureGroupRepo(azureGroupRepo); @@ -87,8 +90,9 @@ private PendingInstance buildPendingInstance(String id, Integer executors) { public void testAwsProvision_whenThereArePendingInsatcnesForAllExecutors_thenShouldNotSacleUp() { String groupId = "sig-1"; BaseSpotinstCloud spotinstCloud = - new AwsSpotinstCloud(groupId, "", "20", "/tmp", null, SlaveUsageEnum.NORMAL, "", false, new SpotReTriggerBuilds(true, false), "", null, - null, null, null, null, null, null, null); + new AwsSpotinstCloud(groupId, "", "20", "/tmp", null, SlaveUsageEnum.NORMAL, "", false, + new SpotReTriggerBuilds(true, false), "", null, null, null, null, null, null, null, + null); Map pendingInstances = new HashMap<>(); pendingInstances.put("sir-1", buildPendingInstance("sir-1", 2)); spotinstCloud.setPendingInstances(pendingInstances); @@ -102,8 +106,9 @@ public void testAwsProvision_whenThereArePendingInsatcnesForPartOfTheExecutors_t String groupId = "sig-1"; String accountId = "act-111"; AwsSpotinstCloud spotinstCloud = - new AwsSpotinstCloud(groupId, "", "20", "/tmp", null, SlaveUsageEnum.NORMAL, "", false, new SpotReTriggerBuilds(true, false), "", null, - null, accountId, null, null, null, null, null); + new AwsSpotinstCloud(groupId, "", "20", "/tmp", null, SlaveUsageEnum.NORMAL, "", false, + new SpotReTriggerBuilds(true, false), "", null, null, accountId, null, null, null, + null, null); jenkinsRule.jenkins.clouds.add(spotinstCloud); Map pendingInstances = new HashMap<>(); @@ -300,8 +305,8 @@ public void testAwCloud_whenUsePrivateIpIsFalse_thenUsePublicIp() { public void testGcpProvision_whenThereArePendingInsatcnesForAllExecutors_thenShouldNotSacleUp() { String groupId = "sig-1"; BaseSpotinstCloud spotinstCloud = - new GcpSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, null, null, null, - null, null, null); + new GcpSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), + "", null, null, null, null, null, null, null, null); Map pendingInstances = new HashMap<>(); pendingInstances.put("sin-1", buildPendingInstance("sin-1", 2)); spotinstCloud.setPendingInstances(pendingInstances); @@ -315,8 +320,8 @@ public void testGcpProvision_whenThereArePendingInsatcnesForPartOfTheExecutors_t String groupId = "sig-1"; String accountId = "act-111"; GcpSpotinstCloud spotinstCloud = - new GcpSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, accountId, null, - null, null, null, null); + new GcpSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), + "", null, null, accountId, null, null, null, null, null); jenkinsRule.jenkins.clouds.add(spotinstCloud); Map pendingInstances = new HashMap<>(); pendingInstances.put("sin-1", buildPendingInstance("sin-1", 2)); @@ -348,9 +353,9 @@ public void testGcpProvision_whenThereArePendingInsatcnesForPartOfTheExecutors_t @Test public void testAzureProvision_whenThereArePendingInsatcnesForAllExecutors_thenShouldNotSacleUp() { String groupId = "sig-1"; - BaseSpotinstCloud spotinstCloud = - new AzureSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(false, false), "", null, null, null, null, - null, null, null, null); + BaseSpotinstCloud spotinstCloud = new AzureSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, + new SpotReTriggerBuilds(false, false), "", null, null, + null, null, null, null, null, null); Map pendingInstances = new HashMap<>(); pendingInstances.put("q3213", buildPendingInstance(groupId, 1)); pendingInstances.put("41234", buildPendingInstance(groupId, 1)); @@ -364,9 +369,9 @@ public void testAzureProvision_whenThereArePendingInsatcnesForAllExecutors_thenS public void testAzureProvision_whenThereArePendingInsatcnesForPartOfTheExecutors_thenShouldSacleUpTheRest() { String groupId = "sig-1"; String accountId = "act-111"; - AzureSpotinstCloud spotinstCloud = - new AzureSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(false, false), "", null, null, accountId, - null, null, null, null, null); + AzureSpotinstCloud spotinstCloud = new AzureSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, + new SpotReTriggerBuilds(false, false), "", null, null, + accountId, null, null, null, null, null); Map pendingInstances = new HashMap<>(); pendingInstances.put("asda", buildPendingInstance(groupId, 1)); pendingInstances.put("ada", buildPendingInstance(groupId, 1)); @@ -393,8 +398,8 @@ public void testAzureProvision_whenThereArePendingInsatcnesForPartOfTheExecutors public void testAzureV3Provision_whenThereArePendingInstancesForAllExecutors_thenShouldNotScaleUp() { String groupId = "sig-1"; BaseSpotinstCloud spotinstCloud = - new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, null, null, null, - null, null, null); + new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", + null, null, null, null, null, null, null, null); Map pendingInstances = new HashMap<>(); pendingInstances.put("vm-1", buildPendingInstance("vm-1", 2)); @@ -410,8 +415,8 @@ public void testAzureV3Provision_whenThereArePendingInstancesForPartOfTheExecuto String groupId = "sig-1"; String accountId = "act-111"; BaseSpotinstCloud spotinstCloud = - new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, accountId, null, - null, null, null, null); + new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", + null, null, accountId, null, null, null, null, null); jenkinsRule.jenkins.clouds.add(spotinstCloud); Map pendingInstances = new HashMap<>(); pendingInstances.put("vm-1", buildPendingInstance("vm-1", 2)); @@ -445,8 +450,8 @@ public void testAzureV3Provision_whenUnrecognizedVmSize_thenDefaultTo1Executor() String groupId = "sig-1"; String accountId = "act-111"; BaseSpotinstCloud spotinstCloud = - new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, accountId, null, - null, null, null, null); + new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", + null, null, accountId, null, null, null, null, null); jenkinsRule.jenkins.clouds.add(spotinstCloud); AzureScaleUpResultNewVm newSpot = new AzureScaleUpResultNewVm(); newSpot.setVmName("vm-2"); @@ -472,8 +477,8 @@ public void testAzureV3Provision_whenNewInstancesAreLaunched_thenTheirSizeIsAcco String groupId = "sig-1"; String accountId = "act-111"; BaseSpotinstCloud spotinstCloud = - new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, accountId, null, - null, null, null, null); + new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", + null, null, accountId, null, null, null, null, null); jenkinsRule.jenkins.clouds.add(spotinstCloud); AzureVmSizeEnum vmSizeBasicA1 = AzureVmSizeEnum.BASIC_A1; AzureVmSizeEnum vmSizeBasicA2 = AzureVmSizeEnum.BASIC_A2; @@ -509,8 +514,8 @@ public void testAzureV3Provision_whenNewInstancesAreLaunched_thenTheirSizeIsAcco String groupId = "sig-1"; String accountId = "act-111"; BaseSpotinstCloud spotinstCloud = - new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, accountId, null, - null, null, null, null); + new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", + null, null, accountId, null, null, null, null, null); jenkinsRule.jenkins.clouds.add(spotinstCloud); Map pendingInstances = new HashMap<>(); pendingInstances.put("vm-1", buildPendingInstance("vm-1", 2)); @@ -548,8 +553,8 @@ public void testAzureV3Provision_whenNewInstancesAreLaunched_thenTheirSizeIsAcco public void testAzureV3Cloud_whenNoConnectionMethodIsProvided_thenDefaultIsJNLP() { String groupId = "sig-1"; BaseSpotinstCloud spotCloud = - new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", null, null, null, null, null, - null, null, null); + new AzureSpotCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(true, false), "", + null, null, null, null, null, null, null, null); jenkinsRule.jenkins.clouds.add(spotCloud); assertEquals(spotCloud.getConnectionMethod(), ConnectionMethodEnum.JNLP); @@ -1275,8 +1280,8 @@ public void testSpotinstSlaveTermination_ifAgentInPendingInstances_thenAgentIsRe Mockito.when( RepoManager.getInstance().getAwsGroupRepo().getGroupInstances(Mockito.anyString(), Mockito.anyString())) .thenReturn(new ApiResponse<>(result)); - Mockito.when( - RepoManager.getInstance().getAwsGroupRepo().getStatefulInstances(Mockito.anyString(), Mockito.anyString())) + Mockito.when(RepoManager.getInstance().getAwsGroupRepo() + .getStatefulInstances(Mockito.anyString(), Mockito.anyString())) .thenReturn(new ApiResponse<>(new LinkedList<>())); List spots = Collections.singletonList(newSpot); @@ -1313,9 +1318,9 @@ public void testSpotinstSlaveTermination_ifAgentInPendingInstances_thenAgentIsRe public void testAzureSpotinstCloud_DescriptorReturnsAzureSpotinstCloudString() { String groupId = "sig-1"; String accountId = "act-111"; - BaseSpotinstCloud spotinstCloud = - new AzureSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, new SpotReTriggerBuilds(false, false), "", null, null, accountId, - null, null, null, null, null); + BaseSpotinstCloud spotinstCloud = new AzureSpotinstCloud(groupId, "", "20", "/tmp", null, "", false, + new SpotReTriggerBuilds(false, false), "", null, null, + accountId, null, null, null, null, null); assertTrue(spotinstCloud.getDescriptor().toString().contains("AzureSpotinstCloud")); } From 26f4c4f4ecc769b279243a0ea51922e0c81ca2e4 Mon Sep 17 00:00:00 2001 From: sitay Date: Tue, 26 Sep 2023 17:14:09 +0300 Subject: [PATCH 11/13] merge deallocation --- .../AwsDeallocateStatefulInstanceRequest.java | 23 ------------------- 1 file changed, 23 deletions(-) delete mode 100644 src/main/java/hudson/plugins/spotinst/model/aws/AwsDeallocateStatefulInstanceRequest.java diff --git a/src/main/java/hudson/plugins/spotinst/model/aws/AwsDeallocateStatefulInstanceRequest.java b/src/main/java/hudson/plugins/spotinst/model/aws/AwsDeallocateStatefulInstanceRequest.java deleted file mode 100644 index 3fc79180..00000000 --- a/src/main/java/hudson/plugins/spotinst/model/aws/AwsDeallocateStatefulInstanceRequest.java +++ /dev/null @@ -1,23 +0,0 @@ -package hudson.plugins.spotinst.model.aws; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; - -/** - * Created by sitay on 30/08/23. - */ -@JsonIgnoreProperties(ignoreUnknown = true) -public class AwsDeallocateStatefulInstanceRequest { - //region members - private AwsStatefulDeallocationConfig statefulDeallocation; - //endregion - - //region getters & setters - public AwsStatefulDeallocationConfig getStatefulDeallocation() { - return statefulDeallocation; - } - - public void setStatefulDeallocation(AwsStatefulDeallocationConfig statefulDeallocation) { - this.statefulDeallocation = statefulDeallocation; - } - //endregion -} From 83b4f88eddfbc80cfcc26faf4e983d6b3df6ac4a Mon Sep 17 00:00:00 2001 From: sitay Date: Wed, 27 Sep 2023 11:13:44 +0300 Subject: [PATCH 12/13] merge deallocation --- .../plugins/spotinst/queue/SpotQueueStatefulTaskDispatcher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/hudson/plugins/spotinst/queue/SpotQueueStatefulTaskDispatcher.java b/src/main/java/hudson/plugins/spotinst/queue/SpotQueueStatefulTaskDispatcher.java index 189fa1ee..d08c020e 100644 --- a/src/main/java/hudson/plugins/spotinst/queue/SpotQueueStatefulTaskDispatcher.java +++ b/src/main/java/hudson/plugins/spotinst/queue/SpotQueueStatefulTaskDispatcher.java @@ -26,7 +26,7 @@ public class SpotQueueStatefulTaskDispatcher extends QueueTaskDispatcher { @CheckForNull public CauseOfBlockage canTake(Node node, Queue.BuildableItem item) { CauseOfBlockage retVal = null; - LOGGER.info("can node {} take item {}", this, item);//TODO: remove + LOGGER.info("can node {} take item {}", node, item);//TODO: remove Queue.Task task = item.task; boolean isTaskReservedForStatefulSlave = task instanceof StatefulInterruptedTask; From ec1bf3195fd345f0003f5b8f61701f455e70320b Mon Sep 17 00:00:00 2001 From: sitay Date: Wed, 27 Sep 2023 12:42:55 +0300 Subject: [PATCH 13/13] merge deallocation --- .../stateful/StatefulInstanceManager.java | 33 ++++++++++++------- .../spotinst/slave/SpotLauncherHelper.java | 23 ++++++------- .../spotinst/slave/SpotinstComputer.java | 16 +++------ 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/src/main/java/hudson/plugins/spotinst/common/stateful/StatefulInstanceManager.java b/src/main/java/hudson/plugins/spotinst/common/stateful/StatefulInstanceManager.java index c47ce3db..9d1c2c1a 100644 --- a/src/main/java/hudson/plugins/spotinst/common/stateful/StatefulInstanceManager.java +++ b/src/main/java/hudson/plugins/spotinst/common/stateful/StatefulInstanceManager.java @@ -6,6 +6,7 @@ import hudson.model.queue.CauseOfBlockage; import hudson.plugins.spotinst.model.aws.stateful.AwsStatefulInstance; import hudson.plugins.spotinst.model.common.BaseStatefulInstance; +import hudson.plugins.spotinst.queue.StatefulInterruptedTask; import hudson.plugins.spotinst.slave.SpotinstSlave; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -22,8 +23,8 @@ public class StatefulInstanceManager { new ConcurrentHashMap<>(); private static final Map ssiByStatefulTask = new ConcurrentHashMap<>(); - private static final Set statefulTasksToReTrigger = - new HashSet<>(); + private static final Map statefulTaskByTask = + new ConcurrentHashMap<>(); //endregion //region methods @@ -59,6 +60,13 @@ public static CauseOfBlockage canNodeTakeStatefulTask(Node node, String stateful return retVal; } + public static String getSsiByTask(Queue.Task task, Executor executor) { + String retVal; + String key = generateKey(task, executor); + retVal = ssiByStatefulTask.get(key); + return retVal; + } + public static void putSsiByTask(Queue.Task task, Executor executor, String ssiId) { String key = generateKey(task, executor); ssiByStatefulTask.put(key, ssiId); @@ -71,22 +79,23 @@ public static String removeSsiByTask(Queue.Task task, Executor executor) { return retVal; } - - public static Boolean isReTriggeringStatefulTask(Queue.Task task, Executor executor) { - boolean retVal; - String key = generateKey(task, executor); - retVal = statefulTasksToReTrigger.contains(key); + public static StatefulInterruptedTask getStatefulTaskByTask(Queue.Task task, Executor executor) { + StatefulInterruptedTask retVal; + String key = generateKey(task, executor); + retVal = statefulTaskByTask.get(key); return retVal; } - public static void handleReTriggeringStatefulTask(Queue.Task task, Executor executor) { + public static void putStatefulTaskByTask(Queue.Task task, Executor executor, StatefulInterruptedTask statefulTask) { String key = generateKey(task, executor); - statefulTasksToReTrigger.add(key); + statefulTaskByTask.put(key, statefulTask); } - public static void handleReTriggeredStatefulTask(Queue.Task task, Executor executor) { - String key = generateKey(task, executor); - statefulTasksToReTrigger.remove(key); + public static StatefulInterruptedTask removeStatefulTaskByTask(Queue.Task task, Executor executor) { + StatefulInterruptedTask retVal; + String key = generateKey(task, executor); + retVal = statefulTaskByTask.remove(key); + return retVal; } //endregion diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java b/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java index 78950867..699ae6ca 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotLauncherHelper.java @@ -56,27 +56,24 @@ static void handleDisconnect(final SlaveComputer computer, Boolean shouldRetrigg synchronized (lock) { - String ssiByTaskName = StatefulInstanceManager.removeSsiByTask(task, executor); + String ssiByTaskName = StatefulInstanceManager.getSsiByTask(task, executor); if (ssiByTaskName != null) { - StatefulInstanceManager.handleReTriggeringStatefulTask(task, executor); StatefulInterruptedTask statefulInterruptedTask = - new StatefulInterruptedTask(ssiByTaskName, task); + StatefulInstanceManager.getStatefulTaskByTask(task, executor); + + if (statefulInterruptedTask == null) { + statefulInterruptedTask = new StatefulInterruptedTask(ssiByTaskName, task); + StatefulInstanceManager.putStatefulTaskByTask(task, executor, statefulInterruptedTask); + } + LOGGER.info(String.format("RETRIGGERING Stateful Task: %s - WITH ACTIONS: %s on SSI %s", statefulInterruptedTask.getTask(), actions, ssiByTaskName)); - Queue.getInstance().schedule2(statefulInterruptedTask, 10, actions); } else { - if (StatefulInstanceManager.isReTriggeringStatefulTask(task, executor)) { - LOGGER.info(String.format( - "Stateful Task : %s is already queued for re-triggering - ignoring it", - task)); - } - else { - LOGGER.info(String.format("RETRIGGERING: %s - WITH ACTIONS: %s", task, actions)); - Queue.getInstance().schedule2(task, 10, actions); - } + LOGGER.info(String.format("RETRIGGERING: %s - WITH ACTIONS: %s", task, actions)); + Queue.getInstance().schedule2(task, 10, actions); } } } diff --git a/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java b/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java index 112908fc..d099fec1 100644 --- a/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java +++ b/src/main/java/hudson/plugins/spotinst/slave/SpotinstComputer.java @@ -85,22 +85,16 @@ public void taskCompleted(Executor executor, Queue.Task task, long durationMS) { if (spotinstCloud != null) { if (spotinstCloud.getStickyNode()) { - String spotinstNodeSsiId = spotinstNode.getSsiId(); - boolean isStatefulNode = StringUtils.isNotEmpty(spotinstNodeSsiId); + String spotinstNodeSsiId = spotinstNode.getSsiId(); + boolean isStatefulNode = StringUtils.isNotEmpty(spotinstNodeSsiId); if (isStatefulNode) { LOGGER.info("task {} accepted on executor {} and is bound to ssi {}", task, executor.getId(), spotinstNodeSsiId); String attachedSsi = StatefulInstanceManager.removeSsiByTask(task, executor); - boolean isTaskReTriggered = StringUtils.isEmpty(attachedSsi); - - if (isTaskReTriggered) { - StatefulInstanceManager.handleReTriggeredStatefulTask(task, executor); - } - else { - LOGGER.info("unbinding stateful task {} and executor {} from ssi {}", task.getName(), - executor.getId(), attachedSsi); - } + LOGGER.info("unbinding stateful task {} and executor {} from ssi {}", task.getName(), + executor.getId(), attachedSsi); + StatefulInstanceManager.removeStatefulTaskByTask(task, executor); } } }