diff --git a/step-controller/step-controller-backend/src/test/resources/step.properties b/step-controller/step-controller-backend/src/test/resources/step.properties index f821e516c1..5082f9e85c 100644 --- a/step-controller/step-controller-backend/src/test/resources/step.properties +++ b/step-controller/step-controller-backend/src/test/resources/step.properties @@ -94,4 +94,6 @@ plugins.FunctionPackagePlugin.maven.repository.central.url=https://repo1.maven.o #timeseries.collections.week.enabled=true #timeseries.collections.week.flush.period=3600000 #timeseries.response.intervals.ideal=100 -#timeseries.response.intervals.max=1000 \ No newline at end of file +#timeseries.response.intervals.max=1000 + +#executions.history.collectCount=10 \ No newline at end of file diff --git a/step-core/src/main/java/step/core/repositories/ImportResult.java b/step-core/src/main/java/step/core/repositories/ImportResult.java index 7df94fd1f4..a5f9dda80e 100644 --- a/step-core/src/main/java/step/core/repositories/ImportResult.java +++ b/step-core/src/main/java/step/core/repositories/ImportResult.java @@ -28,6 +28,7 @@ public class ImportResult implements Serializable { protected boolean successful = false;; protected String planId; + protected String canonicalPlanName; List errors; @@ -47,6 +48,14 @@ public void setPlanId(String planId) { this.planId = planId; } + public String getCanonicalPlanName() { + return canonicalPlanName; + } + + public void setCanonicalPlanName(String canonicalPlanName) { + this.canonicalPlanName = canonicalPlanName; + } + public List getErrors() { return errors; } diff --git a/step-plans/step-plans-base-artefacts/src/main/java/step/reporting/ExecutionHistoryReportPlugin.java b/step-plans/step-plans-base-artefacts/src/main/java/step/reporting/ExecutionHistoryReportPlugin.java new file mode 100644 index 0000000000..5bfe30b486 --- /dev/null +++ b/step-plans/step-plans-base-artefacts/src/main/java/step/reporting/ExecutionHistoryReportPlugin.java @@ -0,0 +1,42 @@ +package step.reporting; + +import ch.exense.commons.app.Configuration; +import step.core.execution.ExecutionContext; +import step.core.execution.model.Execution; +import step.core.execution.model.ExecutionAccessor; +import step.core.execution.model.ExecutionResultSnapshot; +import step.core.plugins.Plugin; +import step.engine.plugins.AbstractExecutionEnginePlugin; + +import java.util.Arrays; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +@Plugin +public class ExecutionHistoryReportPlugin extends AbstractExecutionEnginePlugin { + + public static final String EXECUTIONS_HISTORY_COLLECT_COUNT = "executions.history.collectCount"; + + @Override + public void afterExecutionEnd(ExecutionContext context) { + Configuration configuration = context.getConfiguration(); + Integer countItems = configuration.getPropertyAsInteger(EXECUTIONS_HISTORY_COLLECT_COUNT, 10); + ExecutionAccessor executionAccessor = context.getExecutionAccessor(); + Execution execution = context.getExecutionManager().getExecution(); + long searchBeforeTimestamp = execution.getEndTime() != null ? execution.getEndTime() : System.currentTimeMillis(); + List pastExecutionsSnapshots = executionAccessor.getLastEndedExecutionsByCanonicalPlanName(execution.getImportResult().getCanonicalPlanName(), countItems, searchBeforeTimestamp, Set.of(execution.getId().toString())) + .map(e -> { + ExecutionResultSnapshot snapshot = new ExecutionResultSnapshot(); + snapshot.setId(e.getId().toString()); + snapshot.setResult(e.getResult()); + snapshot.setStatus(e.getStatus()); + snapshot.setStartTime(e.getStartTime()); + return snapshot; + }) + .collect(Collectors.toList()); + execution.setHistoryResults(pastExecutionsSnapshots); + executionAccessor.save(execution); + } + +} diff --git a/step-plans/step-plans-core/src/main/java/step/core/execution/model/Execution.java b/step-plans/step-plans-core/src/main/java/step/core/execution/model/Execution.java index d414cd0ee2..823b7cc7ff 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/execution/model/Execution.java +++ b/step-plans/step-plans-core/src/main/java/step/core/execution/model/Execution.java @@ -56,6 +56,7 @@ public class Execution extends AbstractOrganizableObject implements EnricheableO private String resolvedPlanRootNodeId; private String agentsInvolved; private Version stepVersion; + private List historyResults; public Execution() { super(); @@ -247,6 +248,14 @@ public void setAgentsInvolved(String agentsInvolved) { this.agentsInvolved = agentsInvolved; } + public List getHistoryResults() { + return historyResults; + } + + public void setHistoryResults(List historyResults) { + this.historyResults = historyResults; + } + /** * Returns the version of Step that created this execution. Note that this will be null * for Step versions prior to 3.29.2. diff --git a/step-plans/step-plans-core/src/main/java/step/core/execution/model/ExecutionAccessor.java b/step-plans/step-plans-core/src/main/java/step/core/execution/model/ExecutionAccessor.java index 11c590f014..3b07cd910e 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/execution/model/ExecutionAccessor.java +++ b/step-plans/step-plans-core/src/main/java/step/core/execution/model/ExecutionAccessor.java @@ -21,6 +21,8 @@ import java.util.Date; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; import step.core.accessors.Accessor; import step.core.collections.SearchOrder; @@ -51,4 +53,5 @@ public interface ExecutionAccessor extends Accessor, ExecutionProvide List getLastEndedExecutionsBySchedulerTaskID(String schedulerTaskID, int limit, Long from, Long to); + Stream getLastEndedExecutionsByCanonicalPlanName(String canonicalPlanName, int limit, Long searchBeforeTimestamp, Set excludeExecutionsIds); } diff --git a/step-plans/step-plans-core/src/main/java/step/core/execution/model/ExecutionAccessorImpl.java b/step-plans/step-plans-core/src/main/java/step/core/execution/model/ExecutionAccessorImpl.java index f4733ca29e..fbdabf56c3 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/execution/model/ExecutionAccessorImpl.java +++ b/step-plans/step-plans-core/src/main/java/step/core/execution/model/ExecutionAccessorImpl.java @@ -20,7 +20,9 @@ import java.util.*; import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.apache.commons.collections.CollectionUtils; import step.commons.iterators.SkipLimitIterator; import step.commons.iterators.SkipLimitProvider; import step.core.accessors.AbstractAccessor; @@ -46,6 +48,8 @@ public void createIndexesIfNeeded(Long ttl) { new IndexField("endTime",Order.DESC, null)))); collectionDriver.createOrUpdateCompoundIndex(new LinkedHashSet<>(List.of(new IndexField("planId",Order.ASC, null), new IndexField("endTime",Order.DESC, null)))); + collectionDriver.createOrUpdateCompoundIndex(new LinkedHashSet<>(List.of(new IndexField("canonicalPlanName",Order.ASC, null), + new IndexField("endTime",Order.DESC, null)))); } @Override @@ -174,4 +178,25 @@ public List getLastEndedExecutionsBySchedulerTaskID(String schedulerT order, 0, limit, 0) .collect(Collectors.toList()); } + + @Override + public Stream getLastEndedExecutionsByCanonicalPlanName(String canonicalPlanName, int limit, Long searchBeforeTimestamp, Set excludeExecutionsIds) { + SearchOrder order = new SearchOrder("endTime", -1); + + List filters = new ArrayList<>(List.of( + Filters.equals("importResult.canonicalPlanName", canonicalPlanName), + Filters.equals("status", ExecutionStatus.ENDED.name()) + )); + + if (searchBeforeTimestamp != null) { + filters.add(Filters.lte("endTime", searchBeforeTimestamp)); + } + if (CollectionUtils.isNotEmpty(excludeExecutionsIds)) { + Filter ignoreExecutionsFilter = Filters.in("_id", new ArrayList<>(excludeExecutionsIds)); + filters.add(Filters.not(ignoreExecutionsFilter)); + } + + return collectionDriver + .find(Filters.and(filters), order, 0, limit, 0); + } } diff --git a/step-plans/step-plans-core/src/main/java/step/core/execution/model/ExecutionResultSnapshot.java b/step-plans/step-plans-core/src/main/java/step/core/execution/model/ExecutionResultSnapshot.java new file mode 100644 index 0000000000..c9272dd707 --- /dev/null +++ b/step-plans/step-plans-core/src/main/java/step/core/execution/model/ExecutionResultSnapshot.java @@ -0,0 +1,43 @@ +package step.core.execution.model; + +import step.core.artefacts.reports.ReportNodeStatus; + +public class ExecutionResultSnapshot { + + private String id; + private ExecutionStatus status; + private ReportNodeStatus result; + private long startTime; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public ExecutionStatus getStatus() { + return status; + } + + public void setStatus(ExecutionStatus status) { + this.status = status; + } + + public ReportNodeStatus getResult() { + return result; + } + + public void setResult(ReportNodeStatus result) { + this.result = result; + } + + public long getStartTime() { + return startTime; + } + + public void setStartTime(long startTime) { + this.startTime = startTime; + } +} diff --git a/step-plans/step-plans-core/src/main/java/step/core/repositories/AbstractRepository.java b/step-plans/step-plans-core/src/main/java/step/core/repositories/AbstractRepository.java index f48a4ce163..bb643d9fd0 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/repositories/AbstractRepository.java +++ b/step-plans/step-plans-core/src/main/java/step/core/repositories/AbstractRepository.java @@ -39,6 +39,20 @@ private static Map getCanonicalRepositoryParameters(Set .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)) : null; } + @Override + public String getCanonicalPlanName(Map repositoryParameters) { + if (repositoryParameters == null || repositoryParameters.isEmpty()) { + return null; + } + + return repositoryParameters.entrySet().stream() + .filter(entry -> this.canonicalRepositoryParameters.contains(entry.getKey())) + .sorted(Map.Entry.comparingByKey()) + .map(e -> e.getKey() + "=" + e.getValue()) + .collect(Collectors.joining("&")); + } + + @Override public Set getCanonicalRepositoryParameters() { return canonicalRepositoryParameters; diff --git a/step-plans/step-plans-core/src/main/java/step/core/repositories/Repository.java b/step-plans/step-plans-core/src/main/java/step/core/repositories/Repository.java index ae4598515e..41ec986415 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/repositories/Repository.java +++ b/step-plans/step-plans-core/src/main/java/step/core/repositories/Repository.java @@ -54,4 +54,7 @@ default void postExecution(ExecutionContext context, RepositoryObjectReference r boolean compareCanonicalRepositoryParameters(Map repositoryParameters1, Map repositoryParameters2); Set getCanonicalRepositoryParameters(); + + String getCanonicalPlanName(Map repositoryParameters); + } diff --git a/step-plans/step-plans-core/src/main/java/step/core/repositories/RepositoryObjectManager.java b/step-plans/step-plans-core/src/main/java/step/core/repositories/RepositoryObjectManager.java index 3e8fe9807d..28c2b79318 100644 --- a/step-plans/step-plans-core/src/main/java/step/core/repositories/RepositoryObjectManager.java +++ b/step-plans/step-plans-core/src/main/java/step/core/repositories/RepositoryObjectManager.java @@ -48,7 +48,9 @@ public void registerRepository(String id, Repository repository) { public ImportResult importPlan(ExecutionContext context, RepositoryObjectReference artefact) throws Exception { String respositoryId = artefact.getRepositoryID(); Repository repository = getRepository(respositoryId); - return repository.importArtefact(context, artefact.getRepositoryParameters()); + ImportResult importResult = repository.importArtefact(context, artefact.getRepositoryParameters()); + importResult.setCanonicalPlanName(repository.getCanonicalPlanName(artefact.getRepositoryParameters())); + return importResult; } public Repository getRepository(String respositoryId) {