From 8ba2750e8a4eeb0885bf5265fc0c7a0d856c25d2 Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Wed, 22 Jul 2026 10:43:50 -0400 Subject: [PATCH 01/25] feat: automation initial commit --- src/prerna/project/api/IProject.java | 2 +- .../AutomationCancelledException.java | 47 + .../AutomationConditionEvaluator.java | 446 ++++++ .../automation/AutomationConstants.java | 179 +++ .../automation/AutomationDatabaseUtility.java | 1274 +++++++++++++++++ .../automation/AutomationExecutionUtils.java | 342 +++++ .../CancelAutomationRunReactor.java | 117 ++ .../GetAutomationConfigReactor.java | 100 ++ .../automation/GetAutomationReactor.java | 105 ++ .../automation/GetAutomationRunReactor.java | 131 ++ .../automation/ListAutomationRunsReactor.java | 87 ++ .../automation/PixelExecutionUtils.java | 232 +++ .../ResumeAutomationRunReactor.java | 113 ++ .../automation/RunAutomationNodeReactor.java | 199 +++ .../SaveAutomationConfigReactor.java | 152 ++ .../automation/SaveAutomationReactor.java | 118 ++ .../automation/TriggerAutomationReactor.java | 630 ++++++++ .../nodes/AutomationNodeContext.java | 125 ++ .../nodes/ChildAutomationRunner.java | 70 + .../nodes/DatabaseEngineNodeExecutor.java | 72 + .../automation/nodes/EngineNodeSupport.java | 172 +++ .../nodes/FunctionEngineNodeExecutor.java | 73 + .../nodes/IAutomationNodeExecutor.java | 65 + .../nodes/ModelEngineNodeExecutor.java | 112 ++ .../automation/nodes/NodeDispatcher.java | 60 + .../automation/nodes/PixelNodeExecutor.java | 101 ++ .../nodes/StorageEngineNodeExecutor.java | 110 ++ .../nodes/VectorEngineNodeExecutor.java | 120 ++ .../automation/nodes/WaitNodeExecutor.java | 90 ++ .../reactor/scheduler/SchedulerConstants.java | 2 + .../scheduler/SchedulerOwlCreator.java | 58 + src/prerna/util/SMSSWebWatcher.java | 8 + src/prerna/util/SystemEngineRegistry.java | 4 +- 33 files changed, 5513 insertions(+), 3 deletions(-) create mode 100644 src/prerna/reactor/automation/AutomationCancelledException.java create mode 100644 src/prerna/reactor/automation/AutomationConditionEvaluator.java create mode 100644 src/prerna/reactor/automation/AutomationConstants.java create mode 100644 src/prerna/reactor/automation/AutomationDatabaseUtility.java create mode 100644 src/prerna/reactor/automation/AutomationExecutionUtils.java create mode 100644 src/prerna/reactor/automation/CancelAutomationRunReactor.java create mode 100644 src/prerna/reactor/automation/GetAutomationConfigReactor.java create mode 100644 src/prerna/reactor/automation/GetAutomationReactor.java create mode 100644 src/prerna/reactor/automation/GetAutomationRunReactor.java create mode 100644 src/prerna/reactor/automation/ListAutomationRunsReactor.java create mode 100644 src/prerna/reactor/automation/PixelExecutionUtils.java create mode 100644 src/prerna/reactor/automation/ResumeAutomationRunReactor.java create mode 100644 src/prerna/reactor/automation/RunAutomationNodeReactor.java create mode 100644 src/prerna/reactor/automation/SaveAutomationConfigReactor.java create mode 100644 src/prerna/reactor/automation/SaveAutomationReactor.java create mode 100644 src/prerna/reactor/automation/TriggerAutomationReactor.java create mode 100644 src/prerna/reactor/automation/nodes/AutomationNodeContext.java create mode 100644 src/prerna/reactor/automation/nodes/ChildAutomationRunner.java create mode 100644 src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java create mode 100644 src/prerna/reactor/automation/nodes/EngineNodeSupport.java create mode 100644 src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java create mode 100644 src/prerna/reactor/automation/nodes/IAutomationNodeExecutor.java create mode 100644 src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java create mode 100644 src/prerna/reactor/automation/nodes/NodeDispatcher.java create mode 100644 src/prerna/reactor/automation/nodes/PixelNodeExecutor.java create mode 100644 src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java create mode 100644 src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java create mode 100644 src/prerna/reactor/automation/nodes/WaitNodeExecutor.java diff --git a/src/prerna/project/api/IProject.java b/src/prerna/project/api/IProject.java index cb46971d8e3..3b502f3703c 100644 --- a/src/prerna/project/api/IProject.java +++ b/src/prerna/project/api/IProject.java @@ -58,7 +58,7 @@ public interface IProject extends IEngine, IMCP { String NOTEBOOK_FOLDER = ".notebooks"; enum PROJECT_TYPE { - BLOCKS, CODE, WORKSPACE, SKILL, INSIGHTS, + BLOCKS, CODE, WORKSPACE, SKILL, INSIGHTS, AUTOMATION, }; /** diff --git a/src/prerna/reactor/automation/AutomationCancelledException.java b/src/prerna/reactor/automation/AutomationCancelledException.java new file mode 100644 index 00000000000..c74940fe740 --- /dev/null +++ b/src/prerna/reactor/automation/AutomationCancelledException.java @@ -0,0 +1,47 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +/** + * Thrown mid-node when a cancellation request is detected during a blocking operation. + * + *

Using a distinct unchecked exception type (rather than a flag return value or a checked + * exception) lets nodes that loop internally - such as {@code WaitNodeExecutor} sleeping in + * chunks - abort cleanly without threading a cancellation result through every call frame. The + * caller ({@code TriggerAutomationReactor.executeSingleNode}) catches this type specifically and + * records the run as {@link AutomationConstants#STATUS_CANCELLED} instead of + * {@link AutomationConstants#STATUS_FAILED}, so the end-user sees the correct terminal state. + */ +public class AutomationCancelledException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public AutomationCancelledException(String message) { + super(message); + } +} diff --git a/src/prerna/reactor/automation/AutomationConditionEvaluator.java b/src/prerna/reactor/automation/AutomationConditionEvaluator.java new file mode 100644 index 00000000000..e44ad137de4 --- /dev/null +++ b/src/prerna/reactor/automation/AutomationConditionEvaluator.java @@ -0,0 +1,446 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +/** + * Safe, dependency-free evaluator for automation conditional / while-loop expressions + * and set-variable arithmetic. + * + *

Replaces the previous {@code javax.script} (JavaScript) evaluation. That path + * ran attacker-influenceable data (prior node outputs, HTTP/LLM responses substituted + * into the expression by {@link AutomationExecutionUtils#resolve}) through a scripting + * engine, which is a remote-code-execution vector once any JS engine is on the + * classpath; and on a JDK with no JS engine it silently degraded to an always-true + * check, so conditions never actually evaluated. This evaluator does neither: it + * parses a small, fixed grammar and can only ever return a value - it cannot reach + * Java classes, the filesystem, or the network. + * + *

Supported grammar (after {@code ${var}} substitution): + *

+ * + *

Numeric comparisons are used when both operands parse as numbers; otherwise + * string comparison is used. Truthiness follows the automation convention: a string is + * falsy when empty or equal (ignoring case) to {@code "false"} / {@code "null"} / + * {@code "0"}, truthy otherwise. + */ +public final class AutomationConditionEvaluator { + + private AutomationConditionEvaluator() { + // utility class + } + + /** + * Evaluates {@code expression} and returns its truthiness. If the expression is not + * a parseable expression (e.g. plain free text), falls back to treating the whole + * trimmed string by the truthiness convention rather than failing the node. + * + * @param expression the fully resolved expression (no {@code ${var}} tokens remaining) + * @return the boolean result + */ + public static boolean toBoolean(String expression) { + if (expression == null) { + return false; + } + try { + Object value = new Parser(expression).parse(); + return truthy(value); + } catch (ParseException e) { + // Not an expression we understand - preserve the legacy "truthy string" behavior. + return truthyString(expression.trim()); + } + } + + /** + * Evaluates {@code expression} as arithmetic and returns the numeric result, or + * {@code null} if it is not a pure numeric expression. + * + * @param expression the fully resolved expression + * @return the numeric result, or {@code null} if non-numeric / unparseable + */ + public static Double toNumber(String expression) { + if (expression == null) { + return null; + } + try { + Object value = new Parser(expression).parse(); + return asNumber(value); + } catch (ParseException e) { + return null; + } + } + + // -- Truthiness / coercion ------------------------------------------------------- + + private static boolean truthy(Object v) { + if (v == null) { + return false; + } + if (v instanceof Boolean) { + return (Boolean) v; + } + if (v instanceof Double) { + double d = (Double) v; + return d != 0.0 && !Double.isNaN(d); + } + return truthyString(v.toString().trim()); + } + + private static boolean truthyString(String s) { + return !s.isEmpty() && !"false".equalsIgnoreCase(s) + && !"null".equalsIgnoreCase(s) && !"0".equals(s); + } + + private static Double asNumber(Object v) { + if (v instanceof Double) { + return (Double) v; + } + if (v instanceof Boolean) { + return ((Boolean) v) ? 1.0 : 0.0; + } + if (v instanceof String) { + String s = ((String) v).trim(); + if (s.isEmpty()) { + return null; + } + try { + return Double.parseDouble(s); + } catch (NumberFormatException e) { + return null; + } + } + return null; + } + + private static String asString(Object v) { + return v == null ? "null" : v.toString(); + } + + // -- Recursive-descent parser ---------------------------------------------------- + + /** Thrown internally when the input is not a valid expression. */ + private static final class ParseException extends RuntimeException { + private static final long serialVersionUID = 1L; + + ParseException(String message) { + super(message); + } + } + + private static final class Parser { + + private final String src; + private int pos; + + Parser(String src) { + this.src = src; + } + + Object parse() { + Object result = parseOr(); + skipWhitespace(); + if (this.pos < this.src.length()) { + throw new ParseException("Unexpected trailing input at position " + this.pos); + } + return result; + } + + private Object parseOr() { + Object left = parseAnd(); + while (match("||")) { + boolean l = truthy(left); + Object right = parseAnd(); + left = l || truthy(right); + } + return left; + } + + private Object parseAnd() { + Object left = parseEquality(); + while (match("&&")) { + boolean l = truthy(left); + Object right = parseEquality(); + left = l && truthy(right); + } + return left; + } + + private Object parseEquality() { + Object left = parseRelational(); + while (true) { + if (match("===")) { + left = strictEquals(left, parseRelational()); + } else if (match("!==")) { + left = !strictEquals(left, parseRelational()); + } else if (match("==")) { + left = looseEquals(left, parseRelational()); + } else if (match("!=")) { + left = !looseEquals(left, parseRelational()); + } else { + break; + } + } + return left; + } + + private Object parseRelational() { + Object left = parseAdditive(); + while (true) { + String op = matchAny("<=", ">=", "<", ">"); + if (op == null) { + break; + } + Object right = parseAdditive(); + left = compare(left, right, op); + } + return left; + } + + private Object parseAdditive() { + Object left = parseMultiplicative(); + while (true) { + String op = matchAny("+", "-"); + if (op == null) { + break; + } + Object right = parseMultiplicative(); + Double ln = asNumber(left); + Double rn = asNumber(right); + if ("+".equals(op)) { + // numeric add when both numeric, else string concatenation + left = (ln != null && rn != null) ? (Object) (ln + rn) + : (Object) (asString(left) + asString(right)); + } else { + left = requireNumber(ln, op) - requireNumber(rn, op); + } + } + return left; + } + + private Object parseMultiplicative() { + Object left = parseUnary(); + while (true) { + String op = matchAny("*", "/", "%"); + if (op == null) { + break; + } + double l = requireNumber(asNumber(left), op); + double r = requireNumber(asNumber(parseUnary()), op); + switch (op) { + case "*": left = l * r; break; + case "/": left = l / r; break; + default: left = l % r; break; + } + } + return left; + } + + private Object parseUnary() { + if (match("!")) { + return !truthy(parseUnary()); + } + if (match("-")) { + return -requireNumber(asNumber(parseUnary()), "-"); + } + return parsePrimary(); + } + + private Object parsePrimary() { + skipWhitespace(); + if (this.pos >= this.src.length()) { + throw new ParseException("Unexpected end of expression"); + } + char c = this.src.charAt(this.pos); + if (c == '(') { + this.pos++; + Object inner = parseOr(); + skipWhitespace(); + if (!match(")")) { + throw new ParseException("Expected ')'"); + } + return inner; + } + if (c == '"' || c == '\'') { + return readString(c); + } + if (Character.isDigit(c) || (c == '.' && peekDigit(1))) { + return readNumber(); + } + if (Character.isLetter(c) || c == '_' || c == '$') { + return readWord(); + } + throw new ParseException("Unexpected character '" + c + "' at position " + this.pos); + } + + private Object readString(char quote) { + this.pos++; // opening quote + StringBuilder sb = new StringBuilder(); + while (this.pos < this.src.length()) { + char c = this.src.charAt(this.pos++); + if (c == '\\' && this.pos < this.src.length()) { + char n = this.src.charAt(this.pos++); + switch (n) { + case 'n': sb.append('\n'); break; + case 't': sb.append('\t'); break; + case 'r': sb.append('\r'); break; + default: sb.append(n); break; + } + } else if (c == quote) { + return sb.toString(); + } else { + sb.append(c); + } + } + throw new ParseException("Unterminated string literal"); + } + + private Object readNumber() { + int start = this.pos; + while (this.pos < this.src.length()) { + char c = this.src.charAt(this.pos); + if (Character.isDigit(c) || c == '.' || c == 'e' || c == 'E' + || ((c == '+' || c == '-') && this.pos > start + && (this.src.charAt(this.pos - 1) == 'e' || this.src.charAt(this.pos - 1) == 'E'))) { + this.pos++; + } else { + break; + } + } + try { + return Double.parseDouble(this.src.substring(start, this.pos)); + } catch (NumberFormatException e) { + throw new ParseException("Invalid number literal"); + } + } + + private Object readWord() { + int start = this.pos; + while (this.pos < this.src.length()) { + char c = this.src.charAt(this.pos); + if (Character.isLetterOrDigit(c) || c == '_' || c == '$') { + this.pos++; + } else { + break; + } + } + String word = this.src.substring(start, this.pos); + if ("true".equals(word)) { + return Boolean.TRUE; + } + if ("false".equals(word)) { + return Boolean.FALSE; + } + if ("null".equals(word)) { + return null; + } + return word; // bare word treated as a string operand + } + + // -- token helpers ----------------------------------------------------------- + + private boolean match(String token) { + skipWhitespace(); + if (this.src.startsWith(token, this.pos)) { + this.pos += token.length(); + return true; + } + return false; + } + + private String matchAny(String... tokens) { + for (String t : tokens) { + if (match(t)) { + return t; + } + } + return null; + } + + private void skipWhitespace() { + while (this.pos < this.src.length() && Character.isWhitespace(this.src.charAt(this.pos))) { + this.pos++; + } + } + + private boolean peekDigit(int ahead) { + int i = this.pos + ahead; + return i < this.src.length() && Character.isDigit(this.src.charAt(i)); + } + } + + // -- comparison helpers ---------------------------------------------------------- + + private static double requireNumber(Double d, String op) { + if (d == null) { + throw new ParseException("Operator '" + op + "' requires a numeric operand"); + } + return d; + } + + private static boolean compare(Object left, Object right, String op) { + Double ln = asNumber(left); + Double rn = asNumber(right); + int cmp; + if (ln != null && rn != null) { + cmp = Double.compare(ln, rn); + } else { + cmp = asString(left).compareTo(asString(right)); + } + switch (op) { + case "<": return cmp < 0; + case "<=": return cmp <= 0; + case ">": return cmp > 0; + default: return cmp >= 0; // ">=" + } + } + + private static boolean looseEquals(Object a, Object b) { + Double an = asNumber(a); + Double bn = asNumber(b); + if (an != null && bn != null) { + return an.doubleValue() == bn.doubleValue(); + } + return asString(a).equals(asString(b)); + } + + private static boolean strictEquals(Object a, Object b) { + if (a == null || b == null) { + return a == b; + } + if (a.getClass() != b.getClass()) { + return false; + } + return a.equals(b); + } +} diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java new file mode 100644 index 00000000000..7fa8d450129 --- /dev/null +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -0,0 +1,179 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +/** + * Shared constants for the Automation Engine subsystem. + * Covers table/column names, status values, and node types. + */ +public class AutomationConstants { + + private AutomationConstants() {} + + // -- File names ---------------------------------------------------------------- + + public static final String AUTOMATION_FILE_NAME = "automation.json"; + public static final String AUTOMATION_CONFIG_FILE_NAME = "automation-config.json"; + + /** Placeholder returned by GetAutomationConfig in place of a sensitive value; never persisted back. */ + public static final String SENSITIVE_MASK = "***"; + + // -- DB Table names ------------------------------------------------------------ + + public static final String TABLE_AUTOMATION_RUNS = "AUTOMATION_RUNS"; + public static final String TABLE_AUTOMATION_NODE_OUTPUTS = "AUTOMATION_NODE_OUTPUTS"; + public static final String TABLE_AUTOMATION_FOREACH_ROWS = "AUTOMATION_FOREACH_ROWS"; + /** + * Single-row-per-project marker table enforcing "at most one active run per project" + * cluster-wide, via a primary key on PROJECT_ID. Claiming a row is an atomic INSERT + * (fails with a constraint violation if another run already holds it); the row is + * released on any terminal run status. + */ + public static final String TABLE_AUTOMATION_ACTIVE_RUN = "AUTOMATION_ACTIVE_RUN"; + + // -- AUTOMATION_RUNS columns --------------------------------------------------- + + public static final String RUN_ID = "RUN_ID"; + public static final String PROJECT_ID = "PROJECT_ID"; + public static final String AUTOMATION_ID = "AUTOMATION_ID"; + public static final String STATUS = "STATUS"; + public static final String TRIGGER_TYPE = "TRIGGER_TYPE"; + public static final String RESUMED_FROM_RUN = "RESUMED_FROM_RUN"; + public static final String STARTED_AT = "STARTED_AT"; + public static final String COMPLETED_AT = "COMPLETED_AT"; + public static final String FAILED_NODE_ID = "FAILED_NODE_ID"; + public static final String ERROR_MESSAGE = "ERROR_MESSAGE"; + public static final String LAST_HEARTBEAT = "LAST_HEARTBEAT"; + public static final String TOTAL_NODES = "TOTAL_NODES"; + public static final String COMPLETED_NODES = "COMPLETED_NODES"; + public static final String CREATED_BY = "CREATED_BY"; + public static final String PARENT_RUN_ID = "PARENT_RUN_ID"; + public static final String PARENT_NODE_ID = "PARENT_NODE_ID"; + /** + * Cluster-safe cancellation flag. Set by CancelAutomationRunReactor regardless of which + * pod receives the cancel request; polled by the executing pod's between-node check + * alongside the in-memory (same-pod fast path) AtomicBoolean. + */ + public static final String CANCEL_REQUESTED = "CANCEL_REQUESTED"; + + // -- AUTOMATION_ACTIVE_RUN columns --------------------------------------------- + + public static final String CLAIMED_AT = "CLAIMED_AT"; + + // -- AUTOMATION_NODE_OUTPUTS columns ------------------------------------------ + + public static final String NODE_ID = "NODE_ID"; + public static final String NODE_LABEL = "NODE_LABEL"; + public static final String EXECUTION_ORDER = "EXECUTION_ORDER"; + public static final String DURATION_MS = "DURATION_MS"; + public static final String OUTPUT_VAR = "OUTPUT_VAR"; + public static final String OUTPUT_VALUE = "OUTPUT_VALUE"; + public static final String OUTPUT_PREVIEW = "OUTPUT_PREVIEW"; + public static final String ROW_COUNT = "ROW_COUNT"; + + // -- AUTOMATION_FOREACH_ROWS columns ------------------------------------------ + + public static final String ROW_INDEX = "ROW_INDEX"; + public static final String ROW_KEY = "ROW_KEY"; + + // -- Run statuses -------------------------------------------------------------- + + public static final String STATUS_RUNNING = "RUNNING"; + public static final String STATUS_SUCCESS = "SUCCESS"; + public static final String STATUS_FAILED = "FAILED"; + public static final String STATUS_INTERRUPTED = "INTERRUPTED"; + public static final String STATUS_CANCELLED = "CANCELLED"; + + // -- Node statuses ------------------------------------------------------------- + + public static final String NODE_STATUS_PENDING = "PENDING"; + public static final String NODE_STATUS_RUNNING = "RUNNING"; + public static final String NODE_STATUS_SUCCESS = "SUCCESS"; + public static final String NODE_STATUS_FAILED = "FAILED"; + public static final String NODE_STATUS_SKIPPED = "SKIPPED"; + + // -- Trigger types ------------------------------------------------------------- + + public static final String TRIGGER_MANUAL = "MANUAL"; + public static final String TRIGGER_SCHEDULED = "SCHEDULED"; + public static final String TRIGGER_RESUME = "RESUME"; + public static final String TRIGGER_SUB_AUTOMATION = "SUB_AUTOMATION"; + public static final String TRIGGER_WEBHOOK = "WEBHOOK"; + public static final String TRIGGER_STORAGE_POLL = "STORAGE_POLL"; + public static final String TRIGGER_DB_POLL = "DB_POLL"; + + // -- Node types ---------------------------------------------------------------- + + public static final String NODE_TRIGGER = "trigger"; + public static final String NODE_DATABASE_ENGINE = "database-engine"; + public static final String NODE_STORAGE_ENGINE = "storage-engine"; + public static final String NODE_VECTOR_ENGINE = "vector-engine"; + public static final String NODE_MODEL_ENGINE = "model-engine"; + public static final String NODE_FUNCTION_ENGINE = "function-engine"; + public static final String NODE_APP = "app"; + public static final String NODE_CUSTOM_PIXEL = "custom-pixel"; + public static final String NODE_FOR_EACH = "for-each"; + public static final String NODE_TRANSFORM = "transform"; + public static final String NODE_SUB_AUTOMATION = "sub-automation"; + public static final String NODE_CONDITIONAL = "conditional"; + public static final String NODE_WHILE_LOOP = "while-loop"; + public static final String NODE_TRY_CATCH = "try-catch"; + public static final String NODE_WAIT = "wait"; + public static final String NODE_SET_VARIABLE = "set-variable"; + public static final String NODE_EMAIL = "email"; + public static final String NODE_HTTP_REQUEST = "http-request"; + public static final String NODE_NOTIFICATION = "notification"; + public static final String NODE_SWITCH = "switch"; + public static final String NODE_RETRY = "retry"; + public static final String NODE_PARALLEL = "parallel"; + + // -- Sub-automation node config keys ------------------------------------------ + + public static final String SUB_AUTOMATION_TARGET_PROJECT = "targetProjectId"; + public static final String SUB_AUTOMATION_INPUT_MAPPING = "inputMapping"; + public static final int MAX_SUB_AUTOMATION_DEPTH = 10; + + // -- Data type constants (for table creation) ---------------------------------- + + public static final String VARCHAR_255 = "VARCHAR(255)"; + public static final String VARCHAR_500 = "VARCHAR(500)"; + public static final String VARCHAR_1000 = "VARCHAR(1000)"; + public static final String VARCHAR_2000 = "VARCHAR(2000)"; + public static final String INTEGER = "INTEGER"; + public static final String BIGINT = "BIGINT"; + public static final String NOT_NULL = "NOT NULL"; + + // -- Defaults ------------------------------------------------------------------ + + public static final String DEFAULT_AUTOMATION_ID = "default"; + public static final int DEFAULT_TIMEOUT_SECONDS = 300; + public static final int HEARTBEAT_INTERVAL_SECONDS = 30; + public static final int STALE_HEARTBEAT_THRESHOLD_MINUTES = 5; + public static final int FOREACH_BATCH_SIZE = 100; + public static final int OUTPUT_PREVIEW_MAX_LENGTH = 2000; +} diff --git a/src/prerna/reactor/automation/AutomationDatabaseUtility.java b/src/prerna/reactor/automation/AutomationDatabaseUtility.java new file mode 100644 index 00000000000..e1443b9931f --- /dev/null +++ b/src/prerna/reactor/automation/AutomationDatabaseUtility.java @@ -0,0 +1,1274 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.engine.api.IRDBMSEngine; +import prerna.query.querystruct.SelectQueryStruct; +import prerna.query.querystruct.filters.SimpleQueryFilter; +import prerna.query.querystruct.selectors.QueryColumnOrderBySelector; +import prerna.query.querystruct.selectors.QueryColumnSelector; +import prerna.query.querystruct.selectors.QueryFunctionHelper; +import prerna.query.querystruct.selectors.QueryFunctionSelector; +import prerna.sablecc2.om.PixelDataType; +import prerna.util.ConnectionUtils; +import prerna.util.QueryExecutionUtility; +import prerna.util.SystemEngineRegistry; +import prerna.util.Utility; +import prerna.util.sql.AbstractSqlQueryUtil; + +/** + * Database utility for the Automation Engine subsystem. + * Manages AUTOMATION_RUNS, AUTOMATION_NODE_OUTPUTS, and AUTOMATION_FOREACH_ROWS tables + * in the scheduler database. + * + * Follows the same patterns as {@link prerna.reactor.scheduler.SchedulerDatabaseUtility}. + * Called at platform startup to create tables; provides CRUD for automation execution state. + */ +public final class AutomationDatabaseUtility { + + private static final Logger classLogger = LogManager.getLogger(AutomationDatabaseUtility.class); + + // Table name shortcuts for SelectQueryStruct (TABLE__COLUMN format) + private static final String TABLE_RUNS = AutomationConstants.TABLE_AUTOMATION_RUNS; + private static final String TABLE_NODE_OUTPUTS = AutomationConstants.TABLE_AUTOMATION_NODE_OUTPUTS; + private static final String TABLE_FOREACH = AutomationConstants.TABLE_AUTOMATION_FOREACH_ROWS; + + private AutomationDatabaseUtility() { + // static utility - no instantiation + } + + // -- SQL Statements (INSERT/UPDATE/DELETE - PreparedStatement per SEMOSS conventions) -- + + // AUTOMATION_RUNS + private static final String INSERT_RUN = """ + INSERT INTO AUTOMATION_RUNS \ + (RUN_ID, PROJECT_ID, AUTOMATION_ID, STATUS, TRIGGER_TYPE, RESUMED_FROM_RUN, \ + STARTED_AT, LAST_HEARTBEAT, TOTAL_NODES, COMPLETED_NODES, CREATED_BY, \ + PARENT_RUN_ID, PARENT_NODE_ID) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?)"""; + + private static final String UPDATE_RUN_STATUS = """ + UPDATE AUTOMATION_RUNS SET STATUS = ?, COMPLETED_AT = ?, \ + FAILED_NODE_ID = ?, ERROR_MESSAGE = ? WHERE RUN_ID = ?"""; + + private static final String UPDATE_HEARTBEAT = + "UPDATE AUTOMATION_RUNS SET LAST_HEARTBEAT = ?, COMPLETED_NODES = ? WHERE RUN_ID = ?"; + + private static final String TOUCH_HEARTBEAT = + "UPDATE AUTOMATION_RUNS SET LAST_HEARTBEAT = ? WHERE RUN_ID = ?"; + + private static final String SET_CANCEL_REQUESTED = + "UPDATE AUTOMATION_RUNS SET CANCEL_REQUESTED = ? WHERE RUN_ID = ?"; + + // AUTOMATION_ACTIVE_RUN - single row per project, PK on PROJECT_ID enforces exclusivity + private static final String CLAIM_ACTIVE_RUN = + "INSERT INTO AUTOMATION_ACTIVE_RUN (PROJECT_ID, RUN_ID, CLAIMED_AT) VALUES (?, ?, ?)"; + + private static final String RELEASE_ACTIVE_RUN = + "DELETE FROM AUTOMATION_ACTIVE_RUN WHERE PROJECT_ID = ? AND RUN_ID = ?"; + + private static final String MARK_STALE_INTERRUPTED = """ + UPDATE AUTOMATION_RUNS SET STATUS = ?, COMPLETED_AT = ?, \ + ERROR_MESSAGE = ? WHERE RUN_ID = ?"""; + + // AUTOMATION_NODE_OUTPUTS + private static final String INSERT_NODE_OUTPUT = """ + INSERT INTO AUTOMATION_NODE_OUTPUTS \ + (RUN_ID, NODE_ID, NODE_LABEL, EXECUTION_ORDER, STATUS) \ + VALUES (?, ?, ?, ?, ?)"""; + + private static final String UPDATE_NODE_OUTPUT_SUCCESS = """ + UPDATE AUTOMATION_NODE_OUTPUTS SET STATUS = ?, STARTED_AT = ?, COMPLETED_AT = ?, \ + DURATION_MS = ?, OUTPUT_VAR = ?, OUTPUT_VALUE = ?, OUTPUT_PREVIEW = ?, ROW_COUNT = ? \ + WHERE RUN_ID = ? AND NODE_ID = ?"""; + + private static final String UPDATE_NODE_OUTPUT_FAILED = """ + UPDATE AUTOMATION_NODE_OUTPUTS SET STATUS = ?, STARTED_AT = ?, COMPLETED_AT = ?, \ + DURATION_MS = ?, ERROR_MESSAGE = ? WHERE RUN_ID = ? AND NODE_ID = ?"""; + + private static final String UPDATE_NODE_STATUS = + "UPDATE AUTOMATION_NODE_OUTPUTS SET STATUS = ?, STARTED_AT = ? WHERE RUN_ID = ? AND NODE_ID = ?"; + + // AUTOMATION_FOREACH_ROWS + private static final String INSERT_FOREACH_ROW = """ + INSERT INTO AUTOMATION_FOREACH_ROWS \ + (RUN_ID, NODE_ID, ROW_INDEX, ROW_KEY, STATUS, STARTED_AT, COMPLETED_AT, DURATION_MS, ERROR_MESSAGE) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"""; + + // Aggregate queries (use PreparedStatement - CASE WHEN not easily expressed in SelectQueryStruct) + private static final String SELECT_FOREACH_PROGRESS = """ + SELECT COUNT(*) AS TOTAL, \ + SUM(CASE WHEN STATUS = 'SUCCESS' THEN 1 ELSE 0 END) AS SUCCEEDED, \ + SUM(CASE WHEN STATUS = 'FAILED' THEN 1 ELSE 0 END) AS FAILED \ + FROM AUTOMATION_FOREACH_ROWS WHERE RUN_ID = ? AND NODE_ID = ?"""; + + // -- Initialization ------------------------------------------------------------ + + /** + * Creates automation tables in the scheduler DB if they don't exist. + * Called at platform startup after the scheduler DB is loaded. + * Safe to call on every startup (uses IF NOT EXISTS / metadata checks). + */ + public static void initialize() { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) { + classLogger.warn("Scheduler DB not available - automation tables will not be created"); + return; + } + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + AbstractSqlQueryUtil queryUtil = schedulerDb.getQueryUtil(); + String database = schedulerDb.getDatabase(); + String schema = schedulerDb.getSchema(); + + boolean allowIfExists = queryUtil.allowsIfExistsTableSyntax(); + String dateTimeType = queryUtil.getDateWithTimeDataType(); + String clobType = queryUtil.getClobDataTypeName(); + + createAutomationRunsTable(conn, queryUtil, database, schema, allowIfExists, dateTimeType, clobType); + createAutomationNodeOutputsTable(conn, queryUtil, database, schema, allowIfExists, dateTimeType, clobType); + createAutomationForEachRowsTable(conn, queryUtil, database, schema, allowIfExists, dateTimeType, clobType); + createAutomationActiveRunTable(conn, queryUtil, database, schema, allowIfExists, dateTimeType); + + if (!conn.getAutoCommit()) { + conn.commit(); + } + + classLogger.info("Automation engine tables initialized successfully"); + } catch (Exception e) { + classLogger.error("Failed to initialize automation engine tables: {}", e.getMessage(), e); + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * On startup, marks any runs stuck in RUNNING as INTERRUPTED. + * This handles the case where the server crashed mid-execution. + */ + public static void markStaleRunsInterrupted() { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return; + + // Use SelectQueryStruct to find stale runs + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__RUN_ID", "RUN_ID")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__PROJECT_ID", "PROJECT_ID")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__LAST_HEARTBEAT", "LAST_HEARTBEAT")); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_RUNS + "__STATUS", "==", AutomationConstants.STATUS_RUNNING, PixelDataType.CONST_STRING)); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + if (results == null || results.isEmpty()) { + return; + } + + // For each running run, check if heartbeat is stale and mark as interrupted + Timestamp threshold = toTimestamp(Instant.now().minusSeconds( + AutomationConstants.STALE_HEARTBEAT_THRESHOLD_MINUTES * 60L)); + Timestamp now = toTimestamp(Instant.now()); + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + for (Map row : results) { + String runId = (String) row.get("RUN_ID"); + String projectId = (String) row.get("PROJECT_ID"); + + // Only interrupt runs whose heartbeat is actually stale. A run with a fresh + // heartbeat is still alive (e.g. executing on another node in a cluster), so + // interrupting it would clobber active work. A missing/unparseable heartbeat + // is treated as stale (a crashed run that never checkpointed). + Timestamp lastHeartbeat = toTimestampSafe(row.get("LAST_HEARTBEAT")); + if (lastHeartbeat != null && lastHeartbeat.after(threshold)) { + classLogger.debug("Skipping automation run {} - heartbeat {} is newer than stale threshold {}", + runId, lastHeartbeat, threshold); + continue; + } + + try (PreparedStatement ps = conn.prepareStatement(MARK_STALE_INTERRUPTED)) { + int index = 1; + ps.setString(index++, AutomationConstants.STATUS_INTERRUPTED); + ps.setTimestamp(index++, now); + ps.setString(index++, "Server restarted during execution"); + ps.setString(index++, runId); + int updated = ps.executeUpdate(); + if (updated > 0) { + classLogger.info("Marked stale automation run {} as INTERRUPTED", runId); + } + } + // Release the active-run slot so the project can be re-triggered - otherwise a + // crashed run would permanently block that project from ever running again. + if (projectId != null) { + releaseActiveRun(projectId, runId); + } + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + } catch (Exception e) { + classLogger.error("Failed to mark stale automation runs: {}", e.getMessage(), e); + } finally { + closeConnection(schedulerDb, conn); + } + } + + // -- AUTOMATION_RUNS CRUD -------------------------------------------------------- + + /** + * Checks if an automation already has an active (RUNNING) run for the given project. + * + * @return the active run ID, or null if no run is active + */ + public static String getActiveRun(String projectId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return null; + + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__RUN_ID", "RUN_ID")); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_RUNS + "__PROJECT_ID", "==", projectId, PixelDataType.CONST_STRING)); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_RUNS + "__STATUS", "==", AutomationConstants.STATUS_RUNNING, PixelDataType.CONST_STRING)); + qs.setLimit(1); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + if (results != null && !results.isEmpty()) { + Object runId = results.get(0).get("RUN_ID"); + return runId != null ? runId.toString() : null; + } + return null; + } + + /** + * Atomically claims the "active run" slot for a project. Backed by a single-row-per-project + * marker table ({@code AUTOMATION_ACTIVE_RUN}, PK on {@code PROJECT_ID}) in the shared scheduler + * DB, so this is correct across every pod in a cluster - not just within one JVM. Unlike + * {@link #getActiveRun(String)} (a plain SELECT), this is a single atomic INSERT: a PK + * violation means another run is already active for that project, closing the check-then-insert + * race where two concurrent triggers for the same project could otherwise both start a run and + * double up any node with side effects (e.g. a database-update node running twice). + * + *

If the scheduler DB is unavailable, fails open (returns true) to match the existing + * degraded-mode behavior of the rest of this class (e.g. {@link #insertRun}, which silently + * no-ops when the scheduler DB can't be reached) rather than introduce a new failure mode. + * + * @return true if the slot was claimed (caller may proceed), false if another run already + * holds it for this project + */ + public static boolean claimActiveRun(String projectId, String runId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) { + classLogger.warn("Scheduler DB not available - cannot enforce single-active-run guard for project {}", projectId); + return true; + } + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(CLAIM_ACTIVE_RUN)) { + int index = 1; + ps.setString(index++, projectId); + ps.setString(index++, runId); + ps.setTimestamp(index++, toTimestamp(Instant.now())); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + // Constraint violation (another run already holds this project's slot) is the + // expected/common case here, not an error - log at debug, not error. + classLogger.debug("Could not claim active-run slot for project {} (likely already active): {}", + projectId, e.getMessage()); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Releases the "active run" slot for a project, allowing a new run to be claimed. + * Must be called on every terminal run status (SUCCESS/FAILED/CANCELLED/INTERRUPTED), + * including the stale-run sweep in {@link #markStaleRunsInterrupted()}. + */ + public static boolean releaseActiveRun(String projectId, String runId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(RELEASE_ACTIVE_RUN)) { + ps.setString(1, projectId); + ps.setString(2, runId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to release active-run slot for project {}, run {}: {}", + projectId, runId, e.getMessage(), e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Sets the cluster-safe cancellation flag on a run. Called by {@code CancelAutomationRunReactor} + * regardless of which pod receives the cancel request - unlike the in-memory + * {@code TriggerAutomationReactor.CANCELLATION_FLAGS} map (a same-pod-only fast path), this is + * visible to whichever pod is actually executing the run via {@link #isCancelRequested(String)}. + */ + public static boolean setCancelRequested(String runId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(SET_CANCEL_REQUESTED)) { + ps.setBoolean(1, true); + ps.setString(2, runId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to set cancel-requested flag for run '{}': {}", runId, e.getMessage(), e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Checks the cluster-safe cancellation flag for a run. Polled by the executing pod's + * between-node cancellation check in addition to the local in-memory flag, so a cancel + * request landing on a different pod than the one executing the run is still honored. + */ + public static boolean isCancelRequested(String runId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + AutomationConstants.CANCEL_REQUESTED, + AutomationConstants.CANCEL_REQUESTED)); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_RUNS + "__RUN_ID", "==", runId, PixelDataType.CONST_STRING)); + qs.setLimit(1); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + if (results == null || results.isEmpty()) { + return false; + } + Object flag = results.get(0).get(AutomationConstants.CANCEL_REQUESTED); + if (flag instanceof Boolean) { + return (Boolean) flag; + } + return flag != null && Boolean.parseBoolean(flag.toString()); + } + + /** + * Inserts a new automation run record. + */ + public static boolean insertRun(String runId, String projectId, String automationId, + String triggerType, String resumedFromRun, int totalNodes, String createdBy) { + return insertRun(runId, projectId, automationId, triggerType, resumedFromRun, + totalNodes, createdBy, null, null); + } + + /** + * Inserts a new automation run record, optionally linked to a parent run/node - used when + * a sub-automation node triggers another project's automation. {@code parentRunId} and + * {@code parentNodeId} are null for top-level (manual/scheduled/resume) runs. + */ + public static boolean insertRun(String runId, String projectId, String automationId, + String triggerType, String resumedFromRun, int totalNodes, String createdBy, + String parentRunId, String parentNodeId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + Timestamp now = toTimestamp(Instant.now()); + + try (PreparedStatement ps = conn.prepareStatement(INSERT_RUN)) { + int index = 1; + ps.setString(index++, runId); + ps.setString(index++, projectId); + ps.setString(index++, automationId); + ps.setString(index++, AutomationConstants.STATUS_RUNNING); + ps.setString(index++, triggerType); + setNullableString(ps, index++, resumedFromRun); + ps.setTimestamp(index++, now); + ps.setTimestamp(index++, now); + ps.setInt(index++, totalNodes); + ps.setString(index++, createdBy); + setNullableString(ps, index++, parentRunId); + setNullableString(ps, index++, parentNodeId); + ps.executeUpdate(); + } + + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to insert automation run '{}': {}", runId, e.getMessage(), e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Updates the status of an automation run (on completion or failure). + */ + public static boolean updateRunStatus(String runId, String status, + String failedNodeId, String errorMessage) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(UPDATE_RUN_STATUS)) { + int index = 1; + ps.setString(index++, status); + ps.setTimestamp(index++, toTimestamp(Instant.now())); + setNullableString(ps, index++, failedNodeId); + setNullableString(ps, index++, errorMessage); + ps.setString(index++, runId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to update run status for '{}': {}", runId, e.getMessage(), e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Updates the heartbeat timestamp and completed node count for a running automation. + */ + public static boolean updateHeartbeat(String runId, int completedNodes) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(UPDATE_HEARTBEAT)) { + int index = 1; + ps.setTimestamp(index++, toTimestamp(Instant.now())); + ps.setInt(index++, completedNodes); + ps.setString(index++, runId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to update heartbeat for run '{}': {}", runId, e.getMessage(), e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Updates only the heartbeat timestamp for a running automation. + * Used during long-running for-each batches where completed node count hasn't changed. + */ + public static boolean touchHeartbeat(String runId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(TOUCH_HEARTBEAT)) { + ps.setTimestamp(1, toTimestamp(Instant.now())); + ps.setString(2, runId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to touch heartbeat for run '{}': {}", runId, e.getMessage(), e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Lists automation runs for a project, newest first. + * + * @param projectId the project to query + * @param limit max number of runs to return + * @return list of run summary maps + */ + public static List> getRunsForProject(String projectId, int limit) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return new ArrayList<>(); + + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__RUN_ID", "RUN_ID")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__PROJECT_ID", "PROJECT_ID")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__AUTOMATION_ID", "AUTOMATION_ID")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__STATUS", "STATUS")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__TRIGGER_TYPE", "TRIGGER_TYPE")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__RESUMED_FROM_RUN", "RESUMED_FROM_RUN")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__STARTED_AT", "STARTED_AT")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__COMPLETED_AT", "COMPLETED_AT")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__FAILED_NODE_ID", "FAILED_NODE_ID")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__TOTAL_NODES", "TOTAL_NODES")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__COMPLETED_NODES", "COMPLETED_NODES")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__CREATED_BY", "CREATED_BY")); + + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_RUNS + "__PROJECT_ID", "==", projectId, PixelDataType.CONST_STRING)); + qs.addOrderBy(TABLE_RUNS + "__STARTED_AT", + QueryColumnOrderBySelector.ORDER_BY_DIRECTION.DESC.toString()); + qs.setLimit(limit); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + return results != null ? results : new ArrayList<>(); + } + + /** + * Gets a single run detail by run ID (includes error message). + */ + public static Map getRunDetail(String runId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return null; + + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__RUN_ID", "RUN_ID")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__PROJECT_ID", "PROJECT_ID")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__AUTOMATION_ID", "AUTOMATION_ID")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__STATUS", "STATUS")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__TRIGGER_TYPE", "TRIGGER_TYPE")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__RESUMED_FROM_RUN", "RESUMED_FROM_RUN")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__STARTED_AT", "STARTED_AT")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__COMPLETED_AT", "COMPLETED_AT")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__FAILED_NODE_ID", "FAILED_NODE_ID")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__ERROR_MESSAGE", "ERROR_MESSAGE")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__TOTAL_NODES", "TOTAL_NODES")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__COMPLETED_NODES", "COMPLETED_NODES")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__CREATED_BY", "CREATED_BY")); + + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_RUNS + "__RUN_ID", "==", runId, PixelDataType.CONST_STRING)); + qs.setLimit(1); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + if (results != null && !results.isEmpty()) { + return results.get(0); + } + return null; + } + + // -- AUTOMATION_NODE_OUTPUTS CRUD ------------------------------------------------ + + /** + * Inserts a node output record with PENDING status (before execution). + */ + public static boolean insertNodeOutput(String runId, String nodeId, String nodeLabel, int executionOrder) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(INSERT_NODE_OUTPUT)) { + int index = 1; + ps.setString(index++, runId); + ps.setString(index++, nodeId); + ps.setString(index++, nodeLabel); + ps.setInt(index++, executionOrder); + ps.setString(index++, AutomationConstants.NODE_STATUS_PENDING); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to insert node output for run '{}', node '{}': {}", + runId, nodeId, e.getMessage(), e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Batch-inserts all node outputs for a run (all PENDING). + */ + public static boolean insertAllNodeOutputs(String runId, List> orderedNodes) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(INSERT_NODE_OUTPUT)) { + for (int i = 0; i < orderedNodes.size(); i++) { + Map node = orderedNodes.get(i); + int index = 1; + ps.setString(index++, runId); + ps.setString(index++, (String) node.get("id")); + ps.setString(index++, (String) node.get("label")); + ps.setInt(index++, i); + ps.setString(index++, AutomationConstants.NODE_STATUS_PENDING); + ps.addBatch(); + } + ps.executeBatch(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to batch-insert node outputs for run '{}': {}", runId, e.getMessage(), e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Marks a node as RUNNING (before pixel execution starts). + */ + public static boolean markNodeRunning(String runId, String nodeId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(UPDATE_NODE_STATUS)) { + int index = 1; + ps.setString(index++, AutomationConstants.NODE_STATUS_RUNNING); + ps.setTimestamp(index++, toTimestamp(Instant.now())); + ps.setString(index++, runId); + ps.setString(index++, nodeId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to mark node running for run '{}', node '{}': {}", + runId, nodeId, e.getMessage(), e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Updates a node output after successful execution. + */ + public static boolean updateNodeSuccess(String runId, String nodeId, Timestamp startedAt, + long durationMs, String outputVar, String outputValue, String outputPreview, Integer rowCount) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + AbstractSqlQueryUtil queryUtil = schedulerDb.getQueryUtil(); + + try (PreparedStatement ps = conn.prepareStatement(UPDATE_NODE_OUTPUT_SUCCESS)) { + int index = 1; + ps.setString(index++, AutomationConstants.NODE_STATUS_SUCCESS); + ps.setTimestamp(index++, startedAt); + ps.setTimestamp(index++, toTimestamp(Instant.now())); + ps.setLong(index++, durationMs); + ps.setString(index++, outputVar); + // Handle CLOB for potentially large output values + queryUtil.handleInsertionOfClob(conn, ps, outputValue, index++, AutomationExecutionUtils.GSON); + ps.setString(index++, outputPreview); + if (rowCount != null) { + ps.setInt(index++, rowCount); + } else { + ps.setNull(index++, Types.INTEGER); + } + ps.setString(index++, runId); + ps.setString(index++, nodeId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (Exception e) { + classLogger.error("Failed to update node success for run '{}', node '{}': {}", + runId, nodeId, e.getMessage(), e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Updates a node output after failed execution. + */ + public static boolean updateNodeFailed(String runId, String nodeId, Timestamp startedAt, + long durationMs, String errorMessage) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(UPDATE_NODE_OUTPUT_FAILED)) { + int index = 1; + ps.setString(index++, AutomationConstants.NODE_STATUS_FAILED); + ps.setTimestamp(index++, startedAt); + ps.setTimestamp(index++, toTimestamp(Instant.now())); + ps.setLong(index++, durationMs); + setNullableString(ps, index++, errorMessage); + ps.setString(index++, runId); + ps.setString(index++, nodeId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to update node failed for run '{}', node '{}': {}", + runId, nodeId, e.getMessage(), e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Gets all node outputs for a run (for scope reconstruction during resume). + * + * @return list of node output maps ordered by execution order + */ + public static List> getNodeOutputsForRun(String runId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return new ArrayList<>(); + + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__NODE_ID", "NODE_ID")); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__NODE_LABEL", "NODE_LABEL")); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__EXECUTION_ORDER", "EXECUTION_ORDER")); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__STATUS", "STATUS")); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__STARTED_AT", "STARTED_AT")); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__COMPLETED_AT", "COMPLETED_AT")); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__DURATION_MS", "DURATION_MS")); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__OUTPUT_VAR", "OUTPUT_VAR")); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__OUTPUT_VALUE", "OUTPUT_VALUE")); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__OUTPUT_PREVIEW", "OUTPUT_PREVIEW")); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__ROW_COUNT", "ROW_COUNT")); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__ERROR_MESSAGE", "ERROR_MESSAGE")); + + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_NODE_OUTPUTS + "__RUN_ID", "==", runId, PixelDataType.CONST_STRING)); + qs.addOrderBy(TABLE_NODE_OUTPUTS + "__EXECUTION_ORDER", + QueryColumnOrderBySelector.ORDER_BY_DIRECTION.ASC.toString()); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + return results != null ? results : new ArrayList<>(); + } + + // -- AUTOMATION_FOREACH_ROWS CRUD ------------------------------------------------ + + /** + * Batch-inserts for-each row results. Called with batches of + * {@link AutomationConstants#FOREACH_BATCH_SIZE} rows during for-each execution. + */ + public static boolean insertForEachRowsBatch(String runId, String nodeId, + List rows) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(INSERT_FOREACH_ROW)) { + for (ForEachRowResult row : rows) { + int index = 1; + ps.setString(index++, runId); + ps.setString(index++, nodeId); + ps.setInt(index++, row.rowIndex()); + ps.setString(index++, row.rowKey()); + ps.setString(index++, row.status()); + ps.setTimestamp(index++, row.startedAt()); + ps.setTimestamp(index++, toTimestamp(Instant.now())); + ps.setLong(index++, row.durationMs()); + ps.setString(index++, row.errorMessage()); + ps.addBatch(); + } + ps.executeBatch(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to batch-insert for-each rows for run '{}', node '{}': {}", + runId, nodeId, e.getMessage(), e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Gets aggregate progress for a for-each node. + * Uses PreparedStatement directly because this query involves conditional + * aggregates (SUM with CASE WHEN) not easily expressed via SelectQueryStruct. + * + * @return map with keys "total", "succeeded", "failed" + */ + public static Map getForEachProgress(String runId, String nodeId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + Map progress = new HashMap<>(); + if (schedulerDb == null) return progress; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(SELECT_FOREACH_PROGRESS)) { + ps.setString(1, runId); + ps.setString(2, nodeId); + try (ResultSet rs = ps.executeQuery()) { + if (rs.next()) { + progress.put("total", rs.getInt(1)); + progress.put("succeeded", rs.getInt(2)); + progress.put("failed", rs.getInt(3)); + } + } + } + } catch (SQLException e) { + classLogger.error("Failed to get for-each progress for run '{}', node '{}': {}", + runId, nodeId, e.getMessage(), e); + } finally { + closeConnection(schedulerDb, conn); + } + return progress; + } + + /** + * Gets the failed rows for a for-each node (for drill-down). + */ + public static List> getForEachFailures(String runId, String nodeId, int limit) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return new ArrayList<>(); + + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector(TABLE_FOREACH + "__ROW_INDEX", "ROW_INDEX")); + qs.addSelector(new QueryColumnSelector(TABLE_FOREACH + "__ROW_KEY", "ROW_KEY")); + qs.addSelector(new QueryColumnSelector(TABLE_FOREACH + "__ERROR_MESSAGE", "ERROR_MESSAGE")); + qs.addSelector(new QueryColumnSelector(TABLE_FOREACH + "__COMPLETED_AT", "COMPLETED_AT")); + + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_FOREACH + "__RUN_ID", "==", runId, PixelDataType.CONST_STRING)); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_FOREACH + "__NODE_ID", "==", nodeId, PixelDataType.CONST_STRING)); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_FOREACH + "__STATUS", "==", AutomationConstants.NODE_STATUS_FAILED, PixelDataType.CONST_STRING)); + qs.addOrderBy(TABLE_FOREACH + "__ROW_INDEX", + QueryColumnOrderBySelector.ORDER_BY_DIRECTION.ASC.toString()); + qs.setLimit(limit); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + return results != null ? results : new ArrayList<>(); + } + + /** + * Gets the max row index already processed for a for-each node (for resume). + * + * @return the max row index, or -1 if no rows have been processed + */ + public static int getForEachLastProcessedIndex(String runId, String nodeId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return -1; + + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(QueryFunctionSelector.makeFunctionSelector( + QueryFunctionHelper.MAX, TABLE_FOREACH + "__ROW_INDEX", "MAX_INDEX")); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_FOREACH + "__RUN_ID", "==", runId, PixelDataType.CONST_STRING)); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_FOREACH + "__NODE_ID", "==", nodeId, PixelDataType.CONST_STRING)); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + if (results != null && !results.isEmpty()) { + Object maxVal = results.get(0).get("MAX_INDEX"); + if (maxVal instanceof Number) { + return ((Number) maxVal).intValue(); + } + } + return -1; + } + + // -- Table Creation ------------------------------------------------------------ + + private static void createAutomationRunsTable(Connection conn, AbstractSqlQueryUtil queryUtil, + String database, String schema, boolean allowIfExists, String dateTimeType, String clobType) throws SQLException { + + String tableName = AutomationConstants.TABLE_AUTOMATION_RUNS; + + if (!allowIfExists && queryUtil.tableExists(conn, tableName, database, schema)) { + return; + } + + String[] colNames = { "RUN_ID", "PROJECT_ID", "AUTOMATION_ID", "STATUS", "TRIGGER_TYPE", + "RESUMED_FROM_RUN", "STARTED_AT", "COMPLETED_AT", "FAILED_NODE_ID", + "ERROR_MESSAGE", "LAST_HEARTBEAT", "TOTAL_NODES", "COMPLETED_NODES", "CREATED_BY", + "PARENT_RUN_ID", "PARENT_NODE_ID", "CANCEL_REQUESTED" }; + String[] types = { "VARCHAR(255)", "VARCHAR(255)", "VARCHAR(255)", "VARCHAR(50)", "VARCHAR(50)", + "VARCHAR(255)", dateTimeType, dateTimeType, "VARCHAR(255)", + clobType, dateTimeType, "INTEGER", "INTEGER", "VARCHAR(255)", + "VARCHAR(255)", "VARCHAR(255)", queryUtil.getBooleanDataTypeName() }; + String[] constraints = { "NOT NULL", "NOT NULL", null, "NOT NULL", "NOT NULL", + null, "NOT NULL", null, null, + null, null, null, null, null, + null, null, null }; + + String sql; + if (allowIfExists) { + sql = queryUtil.createTableIfNotExistsWithCustomConstraints(tableName, colNames, types, constraints); + } else { + sql = queryUtil.createTableWithCustomConstraints(tableName, colNames, types, constraints); + } + classLogger.info("Creating table {}: {}", tableName, sql); + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } + + // Migrate installs whose AUTOMATION_RUNS predates sub-automation support / cluster-safe cancel + addColumnIfNotExists(conn, queryUtil, tableName, "PARENT_RUN_ID", "VARCHAR(255)"); + addColumnIfNotExists(conn, queryUtil, tableName, "PARENT_NODE_ID", "VARCHAR(255)"); + addColumnIfNotExists(conn, queryUtil, tableName, "CANCEL_REQUESTED", queryUtil.getBooleanDataTypeName()); + + // Primary key + addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, "PK_AUTOMATION_RUNS", new String[]{"RUN_ID"}); + + // Indexes + createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_AR_PROJECT", tableName, new String[]{"PROJECT_ID"}); + createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_AR_STATUS", tableName, new String[]{"PROJECT_ID", "STATUS"}); + createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_AR_STARTED", tableName, new String[]{"PROJECT_ID", "STARTED_AT"}); + createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_AR_PARENT", tableName, new String[]{"PARENT_RUN_ID"}); + } + + private static void createAutomationNodeOutputsTable(Connection conn, AbstractSqlQueryUtil queryUtil, + String database, String schema, boolean allowIfExists, String dateTimeType, String clobType) throws SQLException { + + String tableName = AutomationConstants.TABLE_AUTOMATION_NODE_OUTPUTS; + + if (!allowIfExists && queryUtil.tableExists(conn, tableName, database, schema)) { + return; + } + + String[] colNames = { "RUN_ID", "NODE_ID", "NODE_LABEL", "EXECUTION_ORDER", "STATUS", + "STARTED_AT", "COMPLETED_AT", "DURATION_MS", "OUTPUT_VAR", + "OUTPUT_VALUE", "OUTPUT_PREVIEW", "ROW_COUNT", "ERROR_MESSAGE" }; + String[] types = { "VARCHAR(255)", "VARCHAR(255)", "VARCHAR(500)", "INTEGER", "VARCHAR(50)", + dateTimeType, dateTimeType, "BIGINT", "VARCHAR(255)", + clobType, "VARCHAR(2000)", "INTEGER", clobType }; + String[] constraints = { "NOT NULL", "NOT NULL", null, "NOT NULL", "NOT NULL", + null, null, null, null, + null, null, null, null }; + + String sql; + if (allowIfExists) { + sql = queryUtil.createTableIfNotExistsWithCustomConstraints(tableName, colNames, types, constraints); + } else { + sql = queryUtil.createTableWithCustomConstraints(tableName, colNames, types, constraints); + } + classLogger.info("Creating table {}: {}", tableName, sql); + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } + + // Composite primary key + addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, "PK_AUTO_NODE_OUT", new String[]{"RUN_ID", "NODE_ID"}); + + // Indexes + createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_ANO_RUN", tableName, new String[]{"RUN_ID"}); + } + + private static void createAutomationForEachRowsTable(Connection conn, AbstractSqlQueryUtil queryUtil, + String database, String schema, boolean allowIfExists, String dateTimeType, String clobType) throws SQLException { + + String tableName = AutomationConstants.TABLE_AUTOMATION_FOREACH_ROWS; + + if (!allowIfExists && queryUtil.tableExists(conn, tableName, database, schema)) { + return; + } + + String[] colNames = { "RUN_ID", "NODE_ID", "ROW_INDEX", "ROW_KEY", "STATUS", + "STARTED_AT", "COMPLETED_AT", "DURATION_MS", "ERROR_MESSAGE" }; + String[] types = { "VARCHAR(255)", "VARCHAR(255)", "INTEGER", "VARCHAR(1000)", "VARCHAR(50)", + dateTimeType, dateTimeType, "BIGINT", clobType }; + String[] constraints = { "NOT NULL", "NOT NULL", "NOT NULL", null, "NOT NULL", + null, null, null, null }; + + String sql; + if (allowIfExists) { + sql = queryUtil.createTableIfNotExistsWithCustomConstraints(tableName, colNames, types, constraints); + } else { + sql = queryUtil.createTableWithCustomConstraints(tableName, colNames, types, constraints); + } + classLogger.info("Creating table {}: {}", tableName, sql); + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } + + // Composite primary key + addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, "PK_AUTO_FE_ROWS", new String[]{"RUN_ID", "NODE_ID", "ROW_INDEX"}); + + // Indexes + createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_AFR_RUN_NODE", tableName, new String[]{"RUN_ID", "NODE_ID"}); + createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_AFR_STATUS", tableName, new String[]{"RUN_ID", "NODE_ID", "STATUS"}); + } + + /** + * Creates the AUTOMATION_ACTIVE_RUN marker table - a single row per project, keyed on + * PROJECT_ID, used to atomically enforce "at most one active run per project" cluster-wide. + * See {@link #claimActiveRun(String, String)} / {@link #releaseActiveRun(String, String)}. + */ + private static void createAutomationActiveRunTable(Connection conn, AbstractSqlQueryUtil queryUtil, + String database, String schema, boolean allowIfExists, String dateTimeType) throws SQLException { + + String tableName = AutomationConstants.TABLE_AUTOMATION_ACTIVE_RUN; + + if (!allowIfExists && queryUtil.tableExists(conn, tableName, database, schema)) { + return; + } + + String[] colNames = { "PROJECT_ID", "RUN_ID", "CLAIMED_AT" }; + String[] types = { "VARCHAR(255)", "VARCHAR(255)", dateTimeType }; + String[] constraints = { "NOT NULL", "NOT NULL", "NOT NULL" }; + + String sql; + if (allowIfExists) { + sql = queryUtil.createTableIfNotExistsWithCustomConstraints(tableName, colNames, types, constraints); + } else { + sql = queryUtil.createTableWithCustomConstraints(tableName, colNames, types, constraints); + } + classLogger.info("Creating table {}: {}", tableName, sql); + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } + + // Primary key on PROJECT_ID alone (not RUN_ID) is what makes claimActiveRun atomic: + // a second INSERT for the same project - from any pod - violates this constraint. + addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, "PK_AUTO_ACTIVE_RUN", new String[]{"PROJECT_ID"}); + } + + // -- Helpers ------------------------------------------------------------------- + + private static IRDBMSEngine getSchedulerDb() { + try { + return SystemEngineRegistry.getSchedulerDb(); + } catch (Exception e) { + classLogger.warn("Could not obtain scheduler DB: {}", e.getMessage()); + return null; + } + } + + private static void closeConnection(IRDBMSEngine engine, Connection conn) { + ConnectionUtils.closeAllConnectionsIfPooling(engine, conn); + } + + /** + * Binds a nullable VARCHAR column value, using {@code setNull(Types.VARCHAR)} instead of + * {@code setString(index, null)} when the value is absent - some JDBC drivers require an + * explicit SQL type for a null bind rather than inferring it from a null String argument. + */ + private static void setNullableString(PreparedStatement ps, int index, String value) throws SQLException { + if (value != null) { + ps.setString(index, value); + } else { + ps.setNull(index, Types.VARCHAR); + } + } + + private static Timestamp toTimestamp(Instant instant) { + return Utility.getSqlTimestampUTC( + LocalDateTime.ofInstant(instant, ZoneOffset.UTC)); + } + + /** + * Best-effort conversion of a value read from the result set into a {@link Timestamp}. + * Handles {@link Timestamp}, any {@link Date}, and parseable timestamp strings. + * Returns null when the value is null or cannot be interpreted. + */ + private static Timestamp toTimestampSafe(Object value) { + if (value == null) { + return null; + } + if (value instanceof Timestamp) { + return (Timestamp) value; + } + if (value instanceof Date) { + return new Timestamp(((Date) value).getTime()); + } + try { + return Timestamp.valueOf(value.toString().trim()); + } catch (IllegalArgumentException e) { + return null; + } + } + + private static void addPrimaryKeyIfNotExists(Connection conn, AbstractSqlQueryUtil queryUtil, + String tableName, String database, String schema, String pkName, String[] columns) { + try { + if (queryUtil.allowIfExistsAddConstraint()) { + String colList = String.join(", ", columns); + String sql = "ALTER TABLE " + tableName + " ADD CONSTRAINT IF NOT EXISTS " + + pkName + " PRIMARY KEY (" + colList + ")"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } + } else { + // Try to add and swallow the error if it already exists + String colList = String.join(", ", columns); + String sql = "ALTER TABLE " + tableName + " ADD CONSTRAINT " + + pkName + " PRIMARY KEY (" + colList + ")"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } + } + } catch (Exception e) { + classLogger.debug("Primary key {} may already exist on {}: {}", pkName, tableName, e.getMessage()); + } + } + + /** + * Adds a column to an existing table if it isn't already present - used to migrate + * AUTOMATION_RUNS for installs that created the table before PARENT_RUN_ID/PARENT_NODE_ID + * existed. Safe to call unconditionally on every startup; errors (column already exists) + * are swallowed just like {@link #addPrimaryKeyIfNotExists}. + */ + private static void addColumnIfNotExists(Connection conn, AbstractSqlQueryUtil queryUtil, + String tableName, String columnName, String columnType) { + try { + String sql = queryUtil.allowIfExistsModifyColumnSyntax() + ? queryUtil.alterTableAddColumnIfNotExists(tableName, columnName, columnType) + : queryUtil.alterTableAddColumn(tableName, columnName, columnType); + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } + } catch (Exception e) { + classLogger.debug("Column {} may already exist on {}: {}", columnName, tableName, e.getMessage()); + } + } + + private static void createIndexIfNotExists(Connection conn, AbstractSqlQueryUtil queryUtil, + boolean allowIfExists, String indexName, String tableName, String[] columns) { + try { + List colList = Arrays.asList(columns); + String sql; + if (allowIfExists && queryUtil.allowIfExistsIndexSyntax()) { + sql = queryUtil.createIndexIfNotExists(indexName, tableName, colList); + } else { + sql = queryUtil.createIndex(indexName, tableName, colList); + } + if (sql != null && !sql.isEmpty()) { + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } + } + } catch (Exception e) { + classLogger.debug("Index {} may already exist on {}: {}", indexName, tableName, e.getMessage()); + } + } + + // -- Data Transfer Object ------------------------------------------------------ + + /** + * Record for a single for-each row result, used in batch inserts. + */ + public record ForEachRowResult( + int rowIndex, + String rowKey, + String status, + String errorMessage, + Timestamp startedAt, + long durationMs + ) { + public ForEachRowResult(int rowIndex, String rowKey, String status, String errorMessage, long startTimeMs) { + this(rowIndex, rowKey, status, errorMessage, + Utility.getSqlTimestampUTC(LocalDateTime.ofInstant( + Instant.ofEpochMilli(startTimeMs), ZoneOffset.UTC)), + System.currentTimeMillis() - startTimeMs); + } + } +} diff --git a/src/prerna/reactor/automation/AutomationExecutionUtils.java b/src/prerna/reactor/automation/AutomationExecutionUtils.java new file mode 100644 index 00000000000..4a66f88d786 --- /dev/null +++ b/src/prerna/reactor/automation/AutomationExecutionUtils.java @@ -0,0 +1,342 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +import prerna.util.AssetUtility; + +/** + * Shared static utilities for the automation execution engine. + * + *

Centralizes logic shared across {@link TriggerAutomationReactor}, + * {@link RunAutomationNodeReactor}, and + * {@link prerna.reactor.automation.foreach.ForEachNodeExecutor}. + */ +public final class AutomationExecutionUtils { + + private static final Logger classLogger = LogManager.getLogger(AutomationExecutionUtils.class); + + /** + * Shared Gson instance for the whole automation engine — public so the + * {@code nodes} sub-package has one shared instance to reuse instead of each + * file declaring its own. + */ + public static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); + + private AutomationExecutionUtils() {} + + /** + * Resolves {@code ${varName}} and {@code ${config.KEY}} placeholders in a template string + * via plain {@link String#replace} — no validation or escaping is applied. + * + *

Any substitution slot whose value can carry user-supplied or LLM-generated text MUST + * be wrapped in {@code ...} in the Pixel template before this is called — + * {@code PixelPreProcessor} handles decoding after parsing, preventing injected content from + * breaking the surrounding Pixel grammar. + */ + public static String resolve(String template, Map scope, Map configMap) { + if (template == null) return ""; + String result = template; + for (Map.Entry e : configMap.entrySet()) { + result = result.replace("${config." + e.getKey() + "}", e.getValue()); + } + for (Map.Entry e : scope.entrySet()) { + if (e.getValue() != null) { + result = result.replace("${" + e.getKey() + "}", e.getValue()); + } + } + return result; + } + + /** + * Returns the per-node timeout from the node definition, defaulting to + * {@link AutomationConstants#DEFAULT_TIMEOUT_SECONDS}. + */ + public static int getNodeTimeout(Map node) { + Object timeout = node.get("timeoutSeconds"); + if (timeout instanceof Number) { + return ((Number) timeout).intValue(); + } + return AutomationConstants.DEFAULT_TIMEOUT_SECONDS; + } + + /** + * Loads {@code automation-config.json} for a project and returns key->value pairs. + * Returns an empty map if the file does not exist or cannot be parsed. + */ + @SuppressWarnings("unchecked") + public static Map loadConfig(String projectId) { + Map map = new HashMap<>(); + try { + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File f = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_CONFIG_FILE_NAME); + if (!f.exists()) return map; + String json = Files.readString(f.toPath(), StandardCharsets.UTF_8); + List> entries = GSON.fromJson(json, + new TypeToken>>() {}.getType()); + if (entries != null) { + for (Map entry : entries) { + String key = (String) entry.get("key"); + String value = (String) entry.get("value"); + if (key != null && value != null) map.put(key, value); + } + } + } catch (Exception e) { + classLogger.warn("Failed to load automation config for project {}: {}", projectId, e.getMessage(), e); + } + return map; + } + + /** + * Applies an output transform to a raw pixel result. + * + *

Supported modes: {@code rows-as-objects}, {@code first-row}, {@code column}, + * {@code jsonpath}. Returns raw serialized JSON when config is null or mode is {@code raw}. + */ + @SuppressWarnings("unchecked") + public static String applyOutputTransform(Object rawResult, Map transformConfig) { + String rawStr = serializeRaw(rawResult); + if (transformConfig == null) return rawStr; + + String mode = (String) transformConfig.getOrDefault("mode", "raw"); + switch (mode) { + case "rows-as-objects": return transformRowsAsObjects(rawStr); + case "first-row": return transformFirstRow(rawStr); + case "column": return transformColumn(rawStr, (String) transformConfig.get("column")); + case "jsonpath": return transformJsonPath(rawStr, (String) transformConfig.get("path")); + default: return rawStr; + } + } + + /** Serializes a raw pixel result to a JSON string. */ + public static String serializeRaw(Object rawResult) { + if (rawResult == null) return ""; + if (rawResult instanceof String) return (String) rawResult; + return GSON.toJson(rawResult); + } + + // -- Private transform helpers ------------------------------------------------- + + @SuppressWarnings("unchecked") + private static String transformRowsAsObjects(String rawStr) { + Map data = extractDataset(parseJson(rawStr)); + if (data == null) return rawStr; + List headers = (List) data.get("headers"); + List> rows = (List>) data.get("values"); + if (headers == null || rows == null) return rawStr; + List> result = new ArrayList<>(); + for (List row : rows) { + Map rowMap = new HashMap<>(); + for (int i = 0; i < headers.size() && i < row.size(); i++) { + rowMap.put(headers.get(i), row.get(i)); + } + result.add(rowMap); + } + return GSON.toJson(result); + } + + @SuppressWarnings("unchecked") + private static String transformFirstRow(String rawStr) { + Map data = extractDataset(parseJson(rawStr)); + if (data == null) return rawStr; + List headers = (List) data.get("headers"); + List> rows = (List>) data.get("values"); + if (headers == null || rows == null || rows.isEmpty()) return rawStr; + Map rowMap = new HashMap<>(); + List first = rows.get(0); + for (int i = 0; i < headers.size() && i < first.size(); i++) { + rowMap.put(headers.get(i), first.get(i)); + } + return GSON.toJson(rowMap); + } + + @SuppressWarnings("unchecked") + private static String transformColumn(String rawStr, String colName) { + if (colName == null || colName.isEmpty()) return rawStr; + Map data = extractDataset(parseJson(rawStr)); + if (data == null) return rawStr; + List headers = (List) data.get("headers"); + List> rows = (List>) data.get("values"); + if (headers == null || rows == null) return rawStr; + int colIdx = headers.indexOf(colName); + if (colIdx < 0) return rawStr; + List col = new ArrayList<>(); + for (List row : rows) col.add(colIdx < row.size() ? row.get(colIdx) : null); + return GSON.toJson(col); + } + + @SuppressWarnings("unchecked") + private static String transformJsonPath(String rawStr, String path) { + if (path == null || path.isEmpty()) return rawStr; + try { + Object current = parseJson(rawStr); + for (String segment : path.split("\\.")) { + if (!(current instanceof Map)) break; + current = ((Map) current).get(segment); + } + if (current == null) return ""; + return current instanceof String ? (String) current : GSON.toJson(current); + } catch (Exception e) { + return rawStr; + } + } + + @SuppressWarnings("unchecked") + private static Map extractDataset(Map parsed) { + if (parsed == null) return null; + if (parsed.containsKey("data") && parsed.get("data") instanceof Map) { + return (Map) parsed.get("data"); + } + if (parsed.containsKey("headers") && parsed.containsKey("values")) return parsed; + return null; + } + + private static Map parseJson(String json) { + if (json == null || json.isBlank()) return null; + try { + return GSON.fromJson(json, new TypeToken>() {}.getType()); + } catch (Exception e) { + return null; + } + } + + // -- Config value coercion ----------------------------------------------------- + + /** + * Returns the trimmed string form of a node config value, or {@code null} if the value is + * {@code null} or blank. + */ + public static String strCfg(Object v) { + return (v != null && !v.toString().isBlank()) ? v.toString() : null; + } + + /** + * Normalizes a config value that may arrive as either an already-parsed {@code Map} or a + * raw JSON string (e.g. {@code inputMapping}, {@code paramValues}). Returns an empty map + * if absent or unparseable. + */ + @SuppressWarnings("unchecked") + public static Map coerceToMap(Object raw) { + if (raw instanceof Map) { + return (Map) raw; + } + if (raw instanceof String str && !str.isBlank()) { + Map parsed = parseJson(str); + if (parsed != null) return parsed; + } + return new HashMap<>(); + } + + // -- Automation document loading ----------------------------------------------- + + /** + * Loads and parses a project's {@code automation.json} (graph + trigger config). + * Throws if the file is missing or unreadable. + */ + @SuppressWarnings("unchecked") + public static Map loadAutomationDoc(String projectId) { + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File f = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); + if (!f.exists()) { + throw new IllegalArgumentException("No automation.json found for this project. Save an automation first."); + } + try { + String json = Files.readString(f.toPath(), StandardCharsets.UTF_8); + return GSON.fromJson(json, new TypeToken>() {}.getType()); + } catch (IOException e) { + throw new IllegalStateException("Failed to read automation.json: " + e.getMessage(), e); + } + } + + // -- Topological sort ---------------------------------------------------------- + + /** + * Topologically sorts a node/edge graph (Kahn's algorithm). Nodes with no incoming edges + * are seeded in node-array order, so a linear automation with no edges still runs + * top-to-bottom in the order the nodes were defined. + */ + @SuppressWarnings("unchecked") + public static List> topoSort(List> nodes, + List> edges) { + if (nodes == null || nodes.isEmpty()) return new ArrayList<>(); + + Map inDegree = new HashMap<>(); + Map> adj = new HashMap<>(); + + for (Map n : nodes) { + String id = (String) n.get("id"); + inDegree.put(id, 0); + adj.put(id, new ArrayList<>()); + } + if (edges != null) { + for (Map e : edges) { + String src = (String) e.get("source"); + String tgt = (String) e.get("target"); + adj.computeIfAbsent(src, k -> new ArrayList<>()).add(tgt); + inDegree.merge(tgt, 1, Integer::sum); + } + } + + Queue queue = new LinkedList<>(); + for (Map n : nodes) { + String id = (String) n.get("id"); + if (inDegree.getOrDefault(id, 0) == 0) queue.add(id); + } + + Map> nodeById = new HashMap<>(); + for (Map n : nodes) nodeById.put((String) n.get("id"), n); + + List> sorted = new ArrayList<>(); + while (!queue.isEmpty()) { + String id = queue.poll(); + sorted.add(nodeById.get(id)); + for (String neighbor : adj.getOrDefault(id, new ArrayList<>())) { + int deg = inDegree.merge(neighbor, -1, Integer::sum); + if (deg == 0) queue.add(neighbor); + } + } + return sorted; + } +} diff --git a/src/prerna/reactor/automation/CancelAutomationRunReactor.java b/src/prerna/reactor/automation/CancelAutomationRunReactor.java new file mode 100644 index 00000000000..fafb84fc620 --- /dev/null +++ b/src/prerna/reactor/automation/CancelAutomationRunReactor.java @@ -0,0 +1,117 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; + +/** + * Requests cancellation of a running automation. Takes effect between nodes (cannot interrupt + * mid-pixel), or mid-wait for nodes that check the flag during blocking operations. + * + *

Pixel: {@code CancelAutomationRun(project=["appId"], runId=["running-run-id"])} + * + *

Sets a cluster-safe cancellation flag ({@code AUTOMATION_RUNS.CANCEL_REQUESTED}, via + * {@link AutomationDatabaseUtility#setCancelRequested(String)}) that the executing pod's + * between-node check polls regardless of which pod that is - this is the source of truth. Also + * attempts an in-memory same-pod signal ({@link TriggerAutomationReactor#requestCancellation(String)}) + * as a fast path when the run happens to be executing on the pod that received this request. The + * run's {@code STATUS} is transitioned to CANCELLED by whichever pod is actually executing it, + * not by this reactor - a truly orphaned run (no pod executing it) is instead caught by the + * periodic stale-heartbeat sweep. + */ +public class CancelAutomationRunReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(CancelAutomationRunReactor.class); + + public CancelAutomationRunReactor() { + this.keysToGet = new String[]{ "project", "runId" }; + this.keyRequired = new int[]{ 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + String runId = this.keyValue.get(this.keysToGet[1]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + if (runId == null || runId.isEmpty()) { + throw new IllegalArgumentException("Must provide the run id to cancel"); + } + + // Auth check + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have edit access"); + } + + // Validate the run exists and is running + Map runDetail = AutomationDatabaseUtility.getRunDetail(runId); + if (runDetail == null) { + throw new IllegalArgumentException("Run not found: " + runId); + } + + String status = (String) runDetail.get(AutomationConstants.STATUS); + if (!AutomationConstants.STATUS_RUNNING.equals(status)) { + throw new IllegalArgumentException( + "Can only cancel RUNNING automations. Current status: " + status); + } + + // Signal cancellation. The cluster-safe DB flag is always set - this is what the pod + // actually executing the run (which may not be this pod) polls between nodes via + // AutomationDatabaseUtility.isCancelRequested(). The in-memory signal is a same-pod fast + // path only. Unlike the prior implementation, we no longer force the run's STATUS to + // CANCELLED when the in-memory signal isn't found on this pod - the run may genuinely + // still be executing on a different pod in a cluster, and overwriting its status here + // would be a lie. The executing pod transitions STATUS to CANCELLED itself once it + // observes the flag; a truly orphaned run (crashed, nobody polling) is caught by the + // periodic stale-heartbeat sweep (AutomationDatabaseUtility.markStaleRunsInterrupted). + boolean signalledLocally = TriggerAutomationReactor.requestCancellation(runId); + AutomationDatabaseUtility.setCancelRequested(runId); + + classLogger.info("Cancel requested for automation run {}: signalledLocally={}", runId, signalledLocally); + + Map result = new HashMap<>(); + result.put(AutomationConstants.RUN_ID, runId); + result.put("cancelRequested", true); + result.put("signalledLocally", signalledLocally); + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } +} diff --git a/src/prerna/reactor/automation/GetAutomationConfigReactor.java b/src/prerna/reactor/automation/GetAutomationConfigReactor.java new file mode 100644 index 00000000000..f931d02eaad --- /dev/null +++ b/src/prerna/reactor/automation/GetAutomationConfigReactor.java @@ -0,0 +1,100 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.AssetUtility; + +public class GetAutomationConfigReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GetAutomationConfigReactor.class); + private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); + + public GetAutomationConfigReactor() { + this.keysToGet = new String[]{ "project" }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File configFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_CONFIG_FILE_NAME); + + if (!configFile.exists()) { + return new NounMetadata(new ArrayList<>(), PixelDataType.VECTOR, PixelOperationType.OPERATION); + } + + try { + String json = Files.readString(configFile.toPath(), StandardCharsets.UTF_8); + // strip sensitive values before returning + List> entries = GSON.fromJson(json, new TypeToken>>() {}.getType()); + if (entries != null) { + for (Map entry : entries) { + Object sensitive = entry.get("sensitive"); + if (Boolean.TRUE.equals(sensitive)) { + entry.put("value", AutomationConstants.SENSITIVE_MASK); + } + } + } + return new NounMetadata(entries != null ? entries : new ArrayList<>(), PixelDataType.VECTOR, PixelOperationType.OPERATION); + } catch (IOException e) { + classLogger.error("Error reading automation config", e); + return new NounMetadata(new ArrayList<>(), PixelDataType.VECTOR, PixelOperationType.OPERATION); + } + } +} diff --git a/src/prerna/reactor/automation/GetAutomationReactor.java b/src/prerna/reactor/automation/GetAutomationReactor.java new file mode 100644 index 00000000000..36c3fde2165 --- /dev/null +++ b/src/prerna/reactor/automation/GetAutomationReactor.java @@ -0,0 +1,105 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.project.api.IProject; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.AssetUtility; +import prerna.util.Utility; + +public class GetAutomationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GetAutomationReactor.class); + private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); + + public GetAutomationReactor() { + this.keysToGet = new String[]{ "project" }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + + IProject project = Utility.getProject(projectId); + if (project.requirePublish(true)) { + classLogger.info("Pulled project {} from cluster", projectId); + } + + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File automationFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); + + if (!automationFile.exists() || !automationFile.isFile()) { + // return empty graph document for brand-new automations + Map empty = new HashMap<>(); + empty.put("version", 1); + Map graph = new HashMap<>(); + graph.put("nodes", new ArrayList<>()); + graph.put("edges", new ArrayList<>()); + empty.put("graph", graph); + return new NounMetadata(empty, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + try { + String json = Files.readString(automationFile.toPath(), StandardCharsets.UTF_8); + Map doc = GSON.fromJson(json, new TypeToken>() {}.getType()); + return new NounMetadata(doc, PixelDataType.MAP, PixelOperationType.OPERATION); + } catch (IOException e) { + classLogger.error("Error reading automation JSON", e); + throw new IllegalArgumentException("Unable to read automation: " + e.getMessage()); + } + } +} diff --git a/src/prerna/reactor/automation/GetAutomationRunReactor.java b/src/prerna/reactor/automation/GetAutomationRunReactor.java new file mode 100644 index 00000000000..7d5c5db3465 --- /dev/null +++ b/src/prerna/reactor/automation/GetAutomationRunReactor.java @@ -0,0 +1,131 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import com.google.gson.Gson; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; + +/** + * Returns detail for a single automation run including per-node results. + * + *

Pixel: {@code GetAutomationRun(app=["appId"], runId=["uuid"])} + * + *

Reads from AUTOMATION_RUNS and AUTOMATION_NODE_OUTPUTS in the scheduler DB. + * Includes for-each progress for batch nodes. + */ +public class GetAutomationRunReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GetAutomationRunReactor.class); + + public GetAutomationRunReactor() { + this.keysToGet = new String[]{ "project", "runId" }; + this.keyRequired = new int[]{ 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + String runId = this.keyValue.get(this.keysToGet[1]); + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + + Map runDetail = AutomationDatabaseUtility.getRunDetail(runId); + if (runDetail == null) { + Map notFound = new HashMap<>(); + notFound.put(AutomationConstants.RUN_ID, runId); + notFound.put("nodeResults", new ArrayList<>()); + return new NounMetadata(notFound, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + // Build node results with for-each progress and while-loop iteration data + List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(runId); + List> nodeResults = new ArrayList<>(); + Gson gson = new Gson(); + + for (Map nodeOutput : nodeOutputs) { + Map nodeResult = new HashMap<>(); + nodeResult.put(AutomationConstants.NODE_ID, nodeOutput.get(AutomationConstants.NODE_ID)); + nodeResult.put(AutomationConstants.NODE_LABEL, nodeOutput.get(AutomationConstants.NODE_LABEL)); + nodeResult.put(AutomationConstants.STATUS, nodeOutput.get(AutomationConstants.STATUS)); + nodeResult.put(AutomationConstants.DURATION_MS, nodeOutput.get(AutomationConstants.DURATION_MS)); + nodeResult.put(AutomationConstants.OUTPUT_PREVIEW, nodeOutput.get(AutomationConstants.OUTPUT_PREVIEW)); + nodeResult.put(AutomationConstants.ERROR_MESSAGE, nodeOutput.get(AutomationConstants.ERROR_MESSAGE)); + + // Include for-each progress if this node has a row count + Object rowCount = nodeOutput.get(AutomationConstants.ROW_COUNT); + if (rowCount != null) { + nodeResult.put(AutomationConstants.ROW_COUNT, rowCount); + String nodeId = (String) nodeOutput.get(AutomationConstants.NODE_ID); + Map progress = AutomationDatabaseUtility.getForEachProgress(runId, nodeId); + if (!progress.isEmpty()) { + nodeResult.put("forEachProgress", progress); + } + } + + // Parse while-loop iteration data stored in OUTPUT_VALUE + Object outputValue = nodeOutput.get(AutomationConstants.OUTPUT_VALUE); + if (outputValue instanceof String) { + String outputStr = (String) outputValue; + if (outputStr.contains("\"__whileResult\":true")) { + try { + @SuppressWarnings("unchecked") + Map wr = gson.fromJson(outputStr, Map.class); + Object iterations = wr.get("iterations"); + if (iterations != null) { + nodeResult.put("iterationResults", iterations); + } + } catch (Exception ignored) { + // malformed JSON - skip + } + } + } + + nodeResults.add(nodeResult); + } + + runDetail.put("nodeResults", nodeResults); + return new NounMetadata(runDetail, PixelDataType.MAP, PixelOperationType.OPERATION); + } +} diff --git a/src/prerna/reactor/automation/ListAutomationRunsReactor.java b/src/prerna/reactor/automation/ListAutomationRunsReactor.java new file mode 100644 index 00000000000..f707829f1da --- /dev/null +++ b/src/prerna/reactor/automation/ListAutomationRunsReactor.java @@ -0,0 +1,87 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; + +/** + * Lists automation run history for a project. + * + *

Pixel: {@code ListAutomationRuns(app=["appId"], limit=["25"])} + * + *

Reads from AUTOMATION_RUNS in the scheduler DB. + */ +public class ListAutomationRunsReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(ListAutomationRunsReactor.class); + + public ListAutomationRunsReactor() { + this.keysToGet = new String[]{ "project", "limit" }; + this.keyRequired = new int[]{ 1, 0 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + String limitStr = this.keyValue.get(this.keysToGet[1]); + int limit = parseLimit(limitStr); + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + + List> runs = AutomationDatabaseUtility.getRunsForProject(projectId, limit); + if (runs == null) { + runs = new ArrayList<>(); + } + + return new NounMetadata(runs, PixelDataType.VECTOR, PixelOperationType.OPERATION); + } + + private int parseLimit(String limitStr) { + if (limitStr == null || limitStr.isEmpty()) return 25; + try { + return Integer.parseInt(limitStr.trim()); + } catch (NumberFormatException e) { + return 25; + } + } +} diff --git a/src/prerna/reactor/automation/PixelExecutionUtils.java b/src/prerna/reactor/automation/PixelExecutionUtils.java new file mode 100644 index 00000000000..fd4b7ca83cf --- /dev/null +++ b/src/prerna/reactor/automation/PixelExecutionUtils.java @@ -0,0 +1,232 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.om.Insight; +import prerna.om.ThreadStore; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.sablecc2.om.task.ITask; + +/** + * Shared utility for executing Pixel expressions and returning clean, serializable results. + * + *

Handles the common pitfalls of raw {@code insight.runPixel()} calls: + *

    + *
  • {@link ITask} materialization - query reactors return lazy cursors that must be collected
    • + *
  • Timeout enforcement - prevents hung queries from blocking pipelines indefinitely
    • + *
  • Error extraction - detects {@link prerna.sablecc2.om.PixelOperationType#ERROR} in results
    • + *
  • Null-safe return - always returns a usable value
    • + *
+ */ +public final class PixelExecutionUtils { + + private static final Logger classLogger = LogManager.getLogger(PixelExecutionUtils.class); + + private PixelExecutionUtils() {} + + /** + * Executes a pixel expression and returns a clean, serializable result. + * + * @param insight the execution context (carries user, varstore, etc.) + * @param pixel the fully resolved pixel string to execute + * @param timeoutSeconds max execution time; 0 or negative means no timeout + * @return the pixel result as a serializable object (Map, List, String, Number, etc.), or null + * @throws AutomationNodeTimeoutException if execution exceeds the timeout + * @throws AutomationPixelException if the pixel returns an error result + */ + public static Object runAndCollect(Insight insight, String pixel, int timeoutSeconds) { + if (pixel == null || pixel.isBlank()) { + return null; + } + + classLogger.debug("Executing pixel (timeout={}s): {}", timeoutSeconds, + pixel.length() > 200 ? pixel.substring(0, 200) + "..." : pixel); + + NounMetadata result; + if (timeoutSeconds > 0) { + result = executeWithTimeout(insight, pixel, timeoutSeconds); + } else { + result = executeDirectly(insight, pixel); + } + + if (result == null) { + return null; + } + + checkForError(result, pixel); + return materializeValue(result); + } + + /** Overload with default timeout from {@link AutomationConstants#DEFAULT_TIMEOUT_SECONDS}. */ + public static Object runAndCollect(Insight insight, String pixel) { + return runAndCollect(insight, pixel, AutomationConstants.DEFAULT_TIMEOUT_SECONDS); + } + + /** Serializes a pixel result to a JSON string for DB storage. */ + public static String serializeResult(Object result) { + if (result == null) return ""; + if (result instanceof String) return (String) result; + return AutomationExecutionUtils.GSON.toJson(result); + } + + /** + * Generates a truncated preview string for quick UI display. + * Returns null if input is null. + */ + public static String generatePreview(String serializedOutput) { + if (serializedOutput == null) return null; + int maxLength = AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH; + return serializedOutput.length() <= maxLength + ? serializedOutput + : serializedOutput.substring(0, maxLength); + } + + // -- Private implementation ---------------------------------------------------- + + private static NounMetadata executeWithTimeout(Insight insight, String pixel, int timeoutSeconds) { + // A new executor is created per timed call and shut down immediately after — no leak. + ExecutorService executor = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "automation-pixel-exec"); + t.setDaemon(true); + return t; + }); + + // ThreadStore is a plain ThreadLocal — not inherited by pool threads. + // Snapshot the caller's context so the worker thread has user/session/insight access. + Map callerContext = ThreadStore.getTheadMapObject(); + final Map contextSnapshot = + callerContext != null ? new HashMap<>(callerContext) : null; + + try { + Callable task = () -> { + if (contextSnapshot != null && !contextSnapshot.isEmpty()) { + ThreadStore.getInsightId(); + ThreadStore.setThreadMapObject(contextSnapshot); + } + try { + return executeDirectly(insight, pixel); + } finally { + ThreadStore.remove(); + } + }; + + Future future = executor.submit(task); + try { + return future.get(timeoutSeconds, TimeUnit.SECONDS); + } catch (TimeoutException e) { + future.cancel(true); + throw new AutomationNodeTimeoutException(pixel, timeoutSeconds); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) throw (RuntimeException) cause; + throw new IllegalStateException("Pixel execution failed: " + cause.getMessage(), cause); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Pixel execution interrupted", e); + } + } finally { + executor.shutdownNow(); + } + } + + private static NounMetadata executeDirectly(Insight insight, String pixel) { + List results = insight.runPixel(pixel).getResults(); + if (results == null || results.isEmpty()) return null; + // For multi-statement pixels the meaningful result is the last one. + return results.get(results.size() - 1); + } + + private static void checkForError(NounMetadata result, String pixel) { + if (result.getOpType() != null && result.getOpType().contains(PixelOperationType.ERROR)) { + String errorMsg = result.getValue() != null ? result.getValue().toString() : "Unknown pixel error"; + throw new AutomationPixelException(pixel, errorMsg); + } + } + + private static Object materializeValue(NounMetadata result) { + Object value = result.getValue(); + if (value == null) return null; + + if (value instanceof ITask) { + classLogger.debug("Materializing ITask result"); + try { + return ((ITask) value).collect(false); + } catch (Exception e) { + classLogger.error("Failed to materialize ITask: {}", e.getMessage(), e); + throw new IllegalStateException("Failed to materialize query result: " + e.getMessage(), e); + } + } + return value; + } + + // -- Exception types ----------------------------------------------------------- + + /** Thrown when a pixel execution exceeds its configured timeout. */ + public static class AutomationNodeTimeoutException extends RuntimeException { + private final int timeoutSeconds; + + public AutomationNodeTimeoutException(String pixel, int timeoutSeconds) { + super("Pixel execution timed out after " + timeoutSeconds + " seconds: " + + (pixel.length() > 100 ? pixel.substring(0, 100) + "..." : pixel)); + this.timeoutSeconds = timeoutSeconds; + } + + public int getTimeoutSeconds() { + return timeoutSeconds; + } + } + + /** Thrown when a pixel returns an ERROR operation type. */ + public static class AutomationPixelException extends RuntimeException { + private final String pixel; + + public AutomationPixelException(String pixel, String errorMessage) { + super(errorMessage); + this.pixel = pixel; + } + + public String getPixel() { + return pixel; + } + } +} diff --git a/src/prerna/reactor/automation/ResumeAutomationRunReactor.java b/src/prerna/reactor/automation/ResumeAutomationRunReactor.java new file mode 100644 index 00000000000..1bcc3649136 --- /dev/null +++ b/src/prerna/reactor/automation/ResumeAutomationRunReactor.java @@ -0,0 +1,113 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; + +/** + * Resumes a failed or interrupted automation run from the first failed node. + * + *

Pixel: {@code ResumeAutomationRun(project=["appId"], runId=["failed-run-id"])} + * + *

Validates the target run exists and is in FAILED or INTERRUPTED status, + * then delegates to {@link TriggerAutomationReactor} with the {@code resumeRunId} + * parameter set. This creates a new run that skips previously successful nodes + * and re-executes from the failure point. + */ +public class ResumeAutomationRunReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(ResumeAutomationRunReactor.class); + + public ResumeAutomationRunReactor() { + this.keysToGet = new String[]{ "project", "runId" }; + this.keyRequired = new int[]{ 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + String runId = this.keyValue.get(this.keysToGet[1]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + if (runId == null || runId.isEmpty()) { + throw new IllegalArgumentException("Must provide the run id to resume"); + } + + // Auth check + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + + // Validate the run exists and is resumable + Map runDetail = AutomationDatabaseUtility.getRunDetail(runId); + if (runDetail == null) { + throw new IllegalArgumentException("Run not found: " + runId); + } + + String status = (String) runDetail.get(AutomationConstants.STATUS); + if (!AutomationConstants.STATUS_FAILED.equals(status) + && !AutomationConstants.STATUS_INTERRUPTED.equals(status)) { + throw new IllegalArgumentException( + "Can only resume FAILED or INTERRUPTED runs. Current status: " + status); + } + + // Verify the run belongs to this project + String runProjectId = (String) runDetail.get(AutomationConstants.PROJECT_ID); + if (!projectId.equals(runProjectId)) { + throw new IllegalArgumentException("Run " + runId + " does not belong to project " + projectId); + } + + classLogger.info("Resuming automation run {} for project {}", runId, projectId); + + // Both values come from validated/DB sources (projectId from testUserProjectIdForAlias, + // runId from AUTOMATION_RUNS), so injection is not expected - guard defensively. + if (projectId.contains("\"") || projectId.contains("]") || + runId.contains("\"") || runId.contains("]")) { + throw new IllegalArgumentException("Invalid characters in project ID or run ID"); + } + + String pixel = "TriggerAutomation(project=[\"" + projectId + "\"], " + + "manual=[\"true\"], resumeRunId=[\"" + runId + "\"]);"; + return new NounMetadata( + PixelExecutionUtils.runAndCollect(this.insight, pixel, 0), + PixelDataType.MAP, PixelOperationType.OPERATION); + } +} diff --git a/src/prerna/reactor/automation/RunAutomationNodeReactor.java b/src/prerna/reactor/automation/RunAutomationNodeReactor.java new file mode 100644 index 00000000000..fbffe2c4a01 --- /dev/null +++ b/src/prerna/reactor/automation/RunAutomationNodeReactor.java @@ -0,0 +1,199 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.time.Instant; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.AssetUtility; + +/** + * Executes a single automation node for testing purposes. + * + *

Pixel: {@code RunAutomationNode(project=["appId"], nodeId=["node-id"], runId=["optional-context-run"])} + * + *

Loads the automation definition, finds the target node, optionally loads scope from a + * prior run's outputs, and executes just that one node. The result is NOT persisted to + * any run - this is a test/preview operation. + * + *

When {@code runId} is provided, prior node outputs from that run are loaded into + * the scope so that {@code ${varName}} references resolve correctly. + */ +public class RunAutomationNodeReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(RunAutomationNodeReactor.class); + private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); + + public RunAutomationNodeReactor() { + this.keysToGet = new String[]{ "project", "nodeId", "runId" }; + this.keyRequired = new int[]{ 1, 1, 0 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + String nodeId = this.keyValue.get(this.keysToGet[1]); + String contextRunId = this.keyValue.get(this.keysToGet[2]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + if (nodeId == null || nodeId.isEmpty()) { + throw new IllegalArgumentException("Must provide a node id"); + } + + // Auth check + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + + // Load automation and find the target node + Map node = findNode(projectId, nodeId); + if (node == null) { + throw new IllegalArgumentException("Node not found in automation: " + nodeId); + } + + // Build scope from context run (if provided) + Map scope = buildScope(contextRunId); + Map configMap = AutomationExecutionUtils.loadConfig(projectId); + + // Execute the node + long startMs = System.currentTimeMillis(); + try { + Object rawOutput = executeNodePixel(node, scope, configMap); + @SuppressWarnings("unchecked") + Map transformConfig = (Map) node.get("outputTransform"); + String transformed = AutomationExecutionUtils.applyOutputTransform(rawOutput, transformConfig); + long durationMs = System.currentTimeMillis() - startMs; + + Map result = new HashMap<>(); + result.put(AutomationConstants.NODE_ID, nodeId); + result.put(AutomationConstants.STATUS, AutomationConstants.NODE_STATUS_SUCCESS); + result.put(AutomationConstants.DURATION_MS, durationMs); + result.put(AutomationConstants.OUTPUT_PREVIEW, PixelExecutionUtils.generatePreview(transformed)); + result.put(AutomationConstants.OUTPUT_VALUE, transformed); + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + + } catch (Exception e) { + long durationMs = System.currentTimeMillis() - startMs; + classLogger.error("Test run of node {} failed: {}", nodeId, e.getMessage(), e); + + Map result = new HashMap<>(); + result.put(AutomationConstants.NODE_ID, nodeId); + result.put(AutomationConstants.STATUS, AutomationConstants.NODE_STATUS_FAILED); + result.put(AutomationConstants.DURATION_MS, durationMs); + result.put(AutomationConstants.ERROR_MESSAGE, e.getMessage()); + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } + } + + // -- Helpers ------------------------------------------------------------------- + + @SuppressWarnings("unchecked") + private Map findNode(String projectId, String nodeId) { + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File f = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); + if (!f.exists()) { + throw new IllegalArgumentException("No automation.json found for this project"); + } + try { + String json = Files.readString(f.toPath(), StandardCharsets.UTF_8); + Map doc = GSON.fromJson(json, new TypeToken>() {}.getType()); + Map graph = (Map) doc.get("graph"); + List> nodes = (List>) graph.get("nodes"); + if (nodes != null) { + for (Map node : nodes) { + if (nodeId.equals(node.get("id"))) { + return node; + } + } + } + } catch (IOException e) { + throw new IllegalStateException("Failed to read automation.json: " + e.getMessage(), e); + } + return null; + } + + private Map buildScope(String contextRunId) { + Map scope = new HashMap<>(); + scope.put("date", Instant.now().toString().substring(0, 10)); + scope.put("triggered_at", Instant.now().toString()); + + if (contextRunId != null && !contextRunId.isEmpty()) { + List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(contextRunId); + for (Map output : nodeOutputs) { + String status = (String) output.get(AutomationConstants.STATUS); + if (AutomationConstants.NODE_STATUS_SUCCESS.equals(status)) { + String outputVar = (String) output.get(AutomationConstants.OUTPUT_VAR); + if (outputVar != null && !outputVar.isEmpty()) { + Object value = output.get(AutomationConstants.OUTPUT_VALUE); + scope.put(outputVar, value != null ? value.toString() : ""); + } + } + } + } + return scope; + } + + private Object executeNodePixel(Map node, Map scope, + Map configMap) { + String type = (String) node.get("type"); + if (AutomationConstants.NODE_TRIGGER.equals(type)) { + return scope.get("triggered_at"); + } + + String builtPixel = (String) node.get("builtPixel"); + if (builtPixel == null || builtPixel.isBlank() || builtPixel.startsWith("//")) { + throw new IllegalStateException("Node has no compiled pixel - save the automation first"); + } + + String resolved = AutomationExecutionUtils.resolve(builtPixel, scope, configMap); + return PixelExecutionUtils.runAndCollect(this.insight, resolved, + AutomationConstants.DEFAULT_TIMEOUT_SECONDS); + } +} diff --git a/src/prerna/reactor/automation/SaveAutomationConfigReactor.java b/src/prerna/reactor/automation/SaveAutomationConfigReactor.java new file mode 100644 index 00000000000..e1e10d92c27 --- /dev/null +++ b/src/prerna/reactor/automation/SaveAutomationConfigReactor.java @@ -0,0 +1,152 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.io.File; +import java.io.IOException; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.AssetUtility; + +public class SaveAutomationConfigReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(SaveAutomationConfigReactor.class); + private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); + + public SaveAutomationConfigReactor() { + this.keysToGet = new String[]{ "project", "config" }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + String configEncoded = this.keyValue.get(this.keysToGet[1]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have edit access"); + } + + String config; + try { + config = URLDecoder.decode(configEncoded != null ? configEncoded : "[]", StandardCharsets.UTF_8); + } catch (Exception e) { + config = configEncoded != null ? configEncoded : "[]"; + } + + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File configFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_CONFIG_FILE_NAME); + + // GetAutomationConfig masks sensitive values (e.g. WEBHOOK_SECRET) as SENSITIVE_MASK. + // Restore the real values from disk so a config round-trip cannot silently destroy them. + config = restoreMaskedSensitiveValues(config, configFile); + + try { + configFile.getParentFile().mkdirs(); + Files.writeString(configFile.toPath(), config, StandardCharsets.UTF_8); + } catch (IOException e) { + classLogger.error("Error saving automation config", e); + throw new IllegalArgumentException("Unable to save automation config: " + e.getMessage()); + } + + SecurityProjectUtils.updateProjectLastEditedDate(projectId); + return new NounMetadata(true, PixelDataType.BOOLEAN, PixelOperationType.OPERATION); + } + + /** + * For every incoming entry marked {@code sensitive} whose value is still the + * {@link AutomationConstants#SENSITIVE_MASK} placeholder, restore the real value from the + * existing on-disk config (matched by {@code key}). Prevents a save from the UI - which + * only ever received the masked value - from overwriting the stored secret. + */ + @SuppressWarnings("unchecked") + private String restoreMaskedSensitiveValues(String incomingJson, File existingFile) { + if (existingFile == null || !existingFile.exists()) { + return incomingJson; + } + try { + List> incoming = GSON.fromJson(incomingJson, + new TypeToken>>() {}.getType()); + String existingJson = Files.readString(existingFile.toPath(), StandardCharsets.UTF_8); + List> existing = GSON.fromJson(existingJson, + new TypeToken>>() {}.getType()); + if (incoming == null || existing == null || existing.isEmpty()) { + return incomingJson; + } + + Map existingValueByKey = new HashMap<>(); + for (Map e : existing) { + existingValueByKey.put(String.valueOf(e.get("key")), e.get("value")); + } + + boolean restoredAny = false; + for (Map entry : incoming) { + if (Boolean.TRUE.equals(entry.get("sensitive")) + && AutomationConstants.SENSITIVE_MASK.equals(entry.get("value"))) { + Object real = existingValueByKey.get(String.valueOf(entry.get("key"))); + if (real != null) { + entry.put("value", real); + restoredAny = true; + } + } + } + return restoredAny ? GSON.toJson(incoming) : incomingJson; + } catch (Exception e) { + // Never let a merge problem overwrite good secrets - keep the existing file untouched. + classLogger.warn("Could not merge sensitive automation config; keeping existing file. Cause: {}", + e.getMessage()); + try { + return Files.readString(existingFile.toPath(), StandardCharsets.UTF_8); + } catch (IOException io) { + return incomingJson; + } + } + } +} diff --git a/src/prerna/reactor/automation/SaveAutomationReactor.java b/src/prerna/reactor/automation/SaveAutomationReactor.java new file mode 100644 index 00000000000..99da53ea6d6 --- /dev/null +++ b/src/prerna/reactor/automation/SaveAutomationReactor.java @@ -0,0 +1,118 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.io.File; +import java.io.IOException; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.cluster.util.ClusterUtil; +import prerna.project.api.IProject; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.AssetUtility; +import prerna.util.Utility; +import prerna.util.git.GitRepoUtils; + +public class SaveAutomationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(SaveAutomationReactor.class); + + public SaveAutomationReactor() { + this.keysToGet = new String[]{ "project", "json" }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + String jsonEncoded = this.keyValue.get(this.keysToGet[1]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + if (jsonEncoded == null || jsonEncoded.isEmpty()) { + throw new IllegalArgumentException("Must provide automation JSON"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have edit access"); + } + + String json; + try { + json = URLDecoder.decode(jsonEncoded, StandardCharsets.UTF_8); + } catch (Exception e) { + json = jsonEncoded; + } + + IProject project = Utility.getProject(projectId); + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File automationFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); + + try { + automationFile.getParentFile().mkdirs(); + Files.writeString(automationFile.toPath(), json, StandardCharsets.UTF_8); + } catch (IOException e) { + classLogger.error("Error saving automation JSON", e); + throw new IllegalArgumentException("Unable to save automation: " + e.getMessage()); + } + + List files = new ArrayList<>(); + files.add(automationFile.getAbsolutePath()); + String versionFolder = AssetUtility.getProjectVersionFolder(project.getProjectName(), projectId); + try { + GitRepoUtils.addSpecificFiles(versionFolder, files); + GitRepoUtils.commitAddedFiles(versionFolder, "Update automation graph", this.insight.getUser()); + } catch (Exception e) { + classLogger.warn("Git commit failed for automation save", e); + } + + if (ClusterUtil.IS_CLUSTER) { + try { + ClusterUtil.pushProjectFolder(project, versionFolder); + } catch (Exception e) { + classLogger.warn("Cluster push failed", e); + } + } + + SecurityProjectUtils.updateProjectLastEditedDate(projectId); + return new NounMetadata(true, PixelDataType.BOOLEAN, PixelOperationType.OPERATION); + } +} diff --git a/src/prerna/reactor/automation/TriggerAutomationReactor.java b/src/prerna/reactor/automation/TriggerAutomationReactor.java new file mode 100644 index 00000000000..8f9583b6ae0 --- /dev/null +++ b/src/prerna/reactor/automation/TriggerAutomationReactor.java @@ -0,0 +1,630 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.om.ThreadStore; +import prerna.reactor.AbstractReactor; +import prerna.reactor.automation.nodes.AutomationNodeContext; +import prerna.reactor.automation.nodes.ChildAutomationRunner; +import prerna.reactor.automation.nodes.DatabaseEngineNodeExecutor; +import prerna.reactor.automation.nodes.FunctionEngineNodeExecutor; +import prerna.reactor.automation.nodes.IAutomationNodeExecutor; +import prerna.reactor.automation.nodes.ModelEngineNodeExecutor; +import prerna.reactor.automation.nodes.NodeDispatcher; +import prerna.reactor.automation.nodes.PixelNodeExecutor; +import prerna.reactor.automation.nodes.StorageEngineNodeExecutor; +import prerna.reactor.automation.nodes.VectorEngineNodeExecutor; +import prerna.reactor.automation.nodes.WaitNodeExecutor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Executes an automation app's graph top-to-bottom with DB-backed state. + * + *

Pixel: {@code TriggerAutomation(project=["appId"], manual=["true"])} + *

Pixel: {@code TriggerAutomation(project=["appId"], resumeRunId=["uuid"])} + * + *

Execution model: + *

    + *
  • Concurrency guard - rejects if a run is already active for this project
    • + *
  • DB checkpoint per node - each completed node is committed immediately
    • + *
  • Stop on error - first node failure halts the pipeline
    • + *
  • Heartbeat - updated every 30s to prove liveness
    • + *
  • Resume - skips nodes that succeeded in a prior run, re-runs from failure
    • + *
+ * + *

State is written to AUTOMATION_RUNS and AUTOMATION_NODE_OUTPUTS in the scheduler DB + * via {@link AutomationDatabaseUtility}. + */ +public class TriggerAutomationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(TriggerAutomationReactor.class); + + /** + * Registry of active run cancellation flags. Keyed by runId. + * When a cancel is requested, the flag is set to true and the executor checks + * between nodes. + */ + private static final ConcurrentHashMap CANCELLATION_FLAGS = new ConcurrentHashMap<>(); + + /** + * Background pool for automation execution. Bounded at 20 concurrent runs with a small queue + * for brief spikes. Rejects beyond capacity so the caller gets an immediate error rather than + * unbounded thread growth. + */ + private static final ExecutorService AUTOMATION_EXECUTOR = new ThreadPoolExecutor( + 2, 20, 60L, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(10), + r -> { + Thread t = new Thread(r, "automation-run-" + System.nanoTime()); + t.setDaemon(true); + return t; + }, + new ThreadPoolExecutor.AbortPolicy() + ); + + /** + * Registry mapping a node's {@code type} to the executor that runs it - replaces the + * previous if/else chain in {@link #executeSingleNode}. Mirrors the existing SEMOSS pattern + * for "one operation, many type-specific implementations, resolved by a type key" + * (see {@code IModelEngine} -> {@code Utility.getModel(engineId)}, {@code IMCP}). + * Executors are stateless and shared across every run/node. + * + * Phase 1 executors only. Phase 2 (conditional, switch, email, http, set-variable, transform, + * retry, try-catch) and Phase 3 (for-each, while-loop, parallel, sub-automation) entries are + * added in their respective bring-over phases. + */ + private static final Map EXECUTORS = Map.of( + AutomationConstants.NODE_WAIT, new WaitNodeExecutor(), + AutomationConstants.NODE_DATABASE_ENGINE, new DatabaseEngineNodeExecutor(), + AutomationConstants.NODE_MODEL_ENGINE, new ModelEngineNodeExecutor(), + AutomationConstants.NODE_VECTOR_ENGINE, new VectorEngineNodeExecutor(), + AutomationConstants.NODE_STORAGE_ENGINE, new StorageEngineNodeExecutor(), + AutomationConstants.NODE_FUNCTION_ENGINE, new FunctionEngineNodeExecutor() + ); + + /** + * Default executor for node types with no dedicated entry above - {@code trigger}, + * {@code app}, and {@code custom-pixel} - which are genuinely arbitrary/composed Pixel with + * no single backing engine. See {@link PixelNodeExecutor}. + */ + private static final IAutomationNodeExecutor PIXEL_EXECUTOR = new PixelNodeExecutor(); + + public TriggerAutomationReactor() { + this.keysToGet = new String[]{ "project", "manual", "resumeRunId", "triggerType" }; + this.keyRequired = new int[]{ 1, 0, 0, 0 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = getProjectId(); + String resumeRunId = this.keyValue.get(this.keysToGet[2]); + + // Determine trigger type + String triggerType = determineTriggerType(resumeRunId); + String userId = getUserId(); + String runId = UUID.randomUUID().toString(); + + // Concurrency guard - atomic claim against the shared scheduler DB, so this is correct + // across every pod in a cluster, not just within this JVM. Prevents two concurrent + // triggers for the same project from both starting a run (which would double up any + // node with side effects, e.g. a database-update node running twice). + if (!AutomationDatabaseUtility.claimActiveRun(projectId, runId)) { + String activeRun = AutomationDatabaseUtility.getActiveRun(projectId); + throw new IllegalArgumentException( + "Automation already has an active run: " + activeRun + + ". Wait for it to complete or cancel it before starting a new run."); + } + + try { + // Load automation definition and config + Map doc = AutomationExecutionUtils.loadAutomationDoc(projectId); + @SuppressWarnings("unchecked") + Map graph = (Map) doc.get("graph"); + @SuppressWarnings("unchecked") + List> nodes = (List>) graph.get("nodes"); + @SuppressWarnings("unchecked") + List> edges = (List>) graph.get("edges"); + Map configMap = AutomationExecutionUtils.loadConfig(projectId); + + // Topological sort + List> ordered = AutomationExecutionUtils.topoSort(nodes, edges); + if (ordered.isEmpty()) { + throw new IllegalArgumentException("Automation has no nodes to execute"); + } + + // Create run record in DB + AutomationDatabaseUtility.insertRun(runId, projectId, AutomationConstants.DEFAULT_AUTOMATION_ID, + triggerType, resumeRunId, ordered.size(), userId); + AutomationDatabaseUtility.insertAllNodeOutputs(runId, ordered); + + // Load prior outputs if resuming + Map priorOutputs = loadPriorOutputs(resumeRunId); + + // Execute nodes on a background thread - an automation can run for hours (large for-each + // ingestion jobs), so it must never block the calling request/websocket thread. + // Progress is checkpointed to AUTOMATION_RUNS/AUTOMATION_NODE_OUTPUTS per node; the + // caller (FE) polls GetAutomationRun(runId) for live status instead of awaiting this. + // Capture the calling thread's ThreadStore (user, session, insight id, scheduler mode) + // so it can be re-seeded on the background executor thread. ThreadStore is a plain + // ThreadLocal and is NOT inherited by pool threads; without this, reactors that read + // ThreadStore during node execution would see null context. + Map parentContext = ThreadStore.getTheadMapObject(); + final Map contextSnapshot = + parentContext != null ? new HashMap<>(parentContext) : null; + + try { + AUTOMATION_EXECUTOR.submit(() -> { + installThreadContext(contextSnapshot); + try { + // executeNodes' own finally always releases the active-run slot - + // including when it throws, which is caught here - so no explicit + // release is needed in this catch block. + executeNodes(runId, projectId, ordered, configMap, priorOutputs); + } catch (Exception e) { + classLogger.error("Unhandled error executing automation run {}: {}", runId, e.getMessage(), e); + AutomationDatabaseUtility.updateRunStatus(runId, + AutomationConstants.STATUS_FAILED, null, e.getMessage()); + } finally { + ThreadStore.remove(); + } + }); + } catch (RejectedExecutionException e) { + // Never submitted - executeNodes' own finally (which normally releases the + // active-run slot) will never run. Release happens in the outer catch below. + AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_FAILED, + null, "Server is at capacity - too many concurrent automation runs"); + throw new IllegalStateException("Too many concurrent automation runs. Please try again shortly."); + } + + Map result = buildRunResult(runId, projectId, AutomationConstants.STATUS_RUNNING, + ordered.size(), 0, null, new ArrayList<>()); + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } catch (RuntimeException e) { + // Any failure before (or in lieu of) the run being successfully handed off to the + // background executor means executeNodes' own finally will never run to release the + // slot - release it here so the project isn't left permanently blocked. + AutomationDatabaseUtility.releaseActiveRun(projectId, runId); + throw e; + } + } + + // -- Core Execution ------------------------------------------------------------ + + private Map executeNodes(String runId, String projectId, + List> ordered, Map configMap, + Map priorOutputs) { + return executeNodes(runId, projectId, ordered, configMap, priorOutputs, + null, Collections.singleton(projectId)); + } + + /** + * Executes an ordered node list for a run. Used both for top-level runs (manual/scheduled/ + * resume, {@code extraInitialScope} null) and for sub-automation calls, where + * {@code extraInitialScope} carries the resolved {@code inputMapping} values and + * {@code ancestorProjectIds} carries the chain of project ids already executing on this + * call stack (self/transitive-call cycle guard). + */ + private Map executeNodes(String runId, String projectId, + List> ordered, Map configMap, + Map priorOutputs, Map extraInitialScope, + Set ancestorProjectIds) { + + // Register cancellation flag + AtomicBoolean cancelled = new AtomicBoolean(false); + CANCELLATION_FLAGS.put(runId, cancelled); + + // Start heartbeat + ScheduledExecutorService heartbeat = startHeartbeat(runId); + + Map scope = buildInitialScope(runId); + if (extraInitialScope != null) { + scope.putAll(extraInitialScope); + } + List> nodeResults = new ArrayList<>(); + int completedCount = 0; + + try { + for (int i = 0; i < ordered.size(); i++) { + Map node = ordered.get(i); + String nodeId = (String) node.get("id"); + String nodeLabel = (String) node.get("label"); + String outputVar = (String) node.get("outputVar"); + String nodeType = (String) node.get("type"); + + // Check cancellation between nodes - the local AtomicBoolean is a same-pod fast + // path (set instantly by CancelAutomationRunReactor when it lands on this pod); + // isCancelRequested() is the cluster-safe source of truth, so a cancel request + // that landed on a different pod than the one executing this run is still + // honored here. + if (cancelled.get() || AutomationDatabaseUtility.isCancelRequested(runId)) { + AutomationDatabaseUtility.updateRunStatus(runId, + AutomationConstants.STATUS_CANCELLED, nodeId, "Run cancelled by user"); + nodeResults.add(buildNodeResult(nodeId, nodeLabel, + AutomationConstants.STATUS_CANCELLED, 0, null, "Run cancelled by user")); + return buildRunResult(runId, projectId, AutomationConstants.STATUS_CANCELLED, + ordered.size(), completedCount, nodeId, nodeResults); + } + + // Resume: skip nodes that already succeeded in the prior run + if (shouldSkipForResume(nodeId, outputVar, priorOutputs, scope)) { + nodeResults.add(buildNodeResult(nodeId, nodeLabel, + AutomationConstants.NODE_STATUS_SKIPPED, 0, + PixelExecutionUtils.generatePreview(priorOutputs.get(nodeId)), null)); + completedCount++; + AutomationDatabaseUtility.updateHeartbeat(runId, completedCount); + continue; + } + + // Execute this node - catching AutomationCancelledException separately so + // mid-node cancellations (e.g. WaitNodeExecutor interrupted mid-sleep) produce + // CANCELLED run status instead of FAILED. + Map nodeResult; + try { + nodeResult = executeSingleNode( + runId, projectId, node, scope, configMap, completedCount, ancestorProjectIds); + } catch (AutomationCancelledException ace) { + AutomationDatabaseUtility.updateRunStatus(runId, + AutomationConstants.STATUS_CANCELLED, nodeId, ace.getMessage()); + nodeResults.add(buildNodeResult(nodeId, nodeLabel, + AutomationConstants.NODE_STATUS_FAILED, 0, null, ace.getMessage())); + return buildRunResult(runId, projectId, AutomationConstants.STATUS_CANCELLED, + ordered.size(), completedCount, nodeId, nodeResults); + } + + String status = (String) nodeResult.get(AutomationConstants.STATUS); + nodeResults.add(nodeResult); + + if (AutomationConstants.NODE_STATUS_SUCCESS.equals(status)) { + // Store output in scope for downstream nodes. + // set-variable nodes write individual variables directly into scope + // inside SetVariableNodeExecutor - skip the generic put to avoid + // overwriting those keys with the JSON blob. + if (outputVar != null && !outputVar.isEmpty() + && !AutomationConstants.NODE_SET_VARIABLE.equals(nodeType)) { + String outputValue = (String) nodeResult.get("outputValue"); + scope.put(outputVar, outputValue != null ? outputValue : ""); + } + completedCount++; + AutomationDatabaseUtility.updateHeartbeat(runId, completedCount); + } else { + // STOP on error + String errorMsg = (String) nodeResult.get(AutomationConstants.ERROR_MESSAGE); + AutomationDatabaseUtility.updateRunStatus(runId, + AutomationConstants.STATUS_FAILED, nodeId, errorMsg); + return buildRunResult(runId, projectId, AutomationConstants.STATUS_FAILED, + ordered.size(), completedCount, nodeId, nodeResults); + } + } + + // All nodes succeeded + AutomationDatabaseUtility.updateRunStatus(runId, + AutomationConstants.STATUS_SUCCESS, null, null); + return buildRunResult(runId, projectId, AutomationConstants.STATUS_SUCCESS, + ordered.size(), completedCount, null, nodeResults); + + } finally { + heartbeat.shutdownNow(); + CANCELLATION_FLAGS.remove(runId); + // Release the cluster-safe active-run slot claimed in execute() (top-level runs) or + // SubAutomationNodeExecutor (sub-automation runs) - covers every terminal path + // (success, failure, cancellation) since they all return through here. + AutomationDatabaseUtility.releaseActiveRun(projectId, runId); + } + } + + private Map executeSingleNode(String runId, String projectId, Map node, + Map scope, Map configMap, int completedCount, + Set ancestorProjectIds) { + + String nodeId = (String) node.get("id"); + String nodeLabel = (String) node.get("label"); + String outputVar = (String) node.get("outputVar"); + String type = (String) node.get("type"); + + // Mark node as running + AutomationDatabaseUtility.markNodeRunning(runId, nodeId); + Timestamp startedAt = toTimestamp(Instant.now()); + long startMs = System.currentTimeMillis(); + + try { + // NodeDispatcher/ChildAutomationRunner recursion callbacks - only composite executors + // (conditional/while-loop/try-catch/switch/retry/parallel) and SubAutomationNodeExecutor + // actually invoke these; every other executor ignores them. + NodeDispatcher nodeDispatcher = (innerNode, innerScope) -> + executeSingleNode(runId, projectId, innerNode, innerScope, configMap, 0, ancestorProjectIds); + ChildAutomationRunner childAutomationRunner = this::executeNodes; + AtomicBoolean cancelFlag = CANCELLATION_FLAGS.get(runId); + + AutomationNodeContext ctx = new AutomationNodeContext(runId, projectId, node, scope, configMap, + ancestorProjectIds, this.insight, cancelFlag, nodeDispatcher, childAutomationRunner); + + IAutomationNodeExecutor executor = EXECUTORS.getOrDefault(type, PIXEL_EXECUTOR); + Object rawOutput = executor.execute(ctx); + + // ForEachNodeExecutor is the one node type whose result map carries a row count for + // the node checkpoint - detected generically by shape (does the returned Map have a + // "totalRows" entry), not by branching on node type. + Integer rowCount = (rawOutput instanceof Map rawMap && rawMap.get("totalRows") instanceof Integer count) + ? count : null; + + // WhileLoopNodeExecutor similarly marks its per-iteration-history result with a + // "__whileResult" entry on the raw (pre-transform) Map, rather than the caller + // string-sniffing already-serialized JSON for a magic key. + Long whileIterationCount = (rawOutput instanceof Map whileMap + && Boolean.TRUE.equals(whileMap.get("__whileResult")) + && whileMap.get("iterationCount") instanceof Number n) + ? n.longValue() : null; + + @SuppressWarnings("unchecked") + Map transformConfig = (Map) node.get("outputTransform"); + String transformed = AutomationExecutionUtils.applyOutputTransform(rawOutput, transformConfig); + + long durationMs = System.currentTimeMillis() - startMs; + String preview = whileIterationCount != null + ? whileIterationCount + " iteration" + (whileIterationCount == 1 ? "" : "s") + : PixelExecutionUtils.generatePreview(transformed); + + AutomationDatabaseUtility.updateNodeSuccess(runId, nodeId, startedAt, + durationMs, outputVar, transformed, preview, rowCount); + + Map result = buildNodeResult(nodeId, nodeLabel, + AutomationConstants.NODE_STATUS_SUCCESS, durationMs, preview, null); + result.put("outputValue", transformed); + if (rowCount != null) { + result.put(AutomationConstants.ROW_COUNT, rowCount); + } + return result; + + } catch (AutomationCancelledException ace) { + long durationMs = System.currentTimeMillis() - startMs; + // Mid-node cancellation (e.g. WaitNodeExecutor interrupted mid-sleep). + // Update node as failed since it didn't complete, then propagate so executeNodes + // records the run as CANCELLED rather than FAILED. + AutomationDatabaseUtility.updateNodeFailed(runId, nodeId, startedAt, durationMs, ace.getMessage()); + throw ace; + } catch (Exception e) { + long durationMs = System.currentTimeMillis() - startMs; + String errorMsg = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); + classLogger.error("Node {} ({}) failed: {}", nodeId, nodeLabel, errorMsg, e); + + AutomationDatabaseUtility.updateNodeFailed(runId, nodeId, startedAt, durationMs, errorMsg); + + return buildNodeResult(nodeId, nodeLabel, + AutomationConstants.NODE_STATUS_FAILED, durationMs, null, errorMsg); + } + } + + // -- Resume Logic -------------------------------------------------------------- + + private Map loadPriorOutputs(String resumeRunId) { + Map outputs = new HashMap<>(); + if (resumeRunId == null || resumeRunId.isEmpty()) { + return outputs; + } + + List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(resumeRunId); + for (Map nodeOutput : nodeOutputs) { + String status = (String) nodeOutput.get(AutomationConstants.STATUS); + if (AutomationConstants.NODE_STATUS_SUCCESS.equals(status)) { + String nodeId = (String) nodeOutput.get(AutomationConstants.NODE_ID); + String outputValue = nodeOutput.get(AutomationConstants.OUTPUT_VALUE) != null + ? nodeOutput.get(AutomationConstants.OUTPUT_VALUE).toString() : ""; + outputs.put(nodeId, outputValue); + } + } + return outputs; + } + + private boolean shouldSkipForResume(String nodeId, String outputVar, + Map priorOutputs, Map scope) { + if (priorOutputs.isEmpty() || !priorOutputs.containsKey(nodeId)) { + return false; + } + // Copy prior output to scope so downstream nodes can reference it + String priorValue = priorOutputs.get(nodeId); + if (outputVar != null && !outputVar.isEmpty()) { + scope.put(outputVar, priorValue); + } + return true; + } + + // -- Heartbeat ----------------------------------------------------------------- + + private ScheduledExecutorService startHeartbeat(String runId) { + ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "automation-heartbeat-" + runId.substring(0, 8)); + t.setDaemon(true); + return t; + }); + // Heartbeat fires every 30 seconds - just proves liveness; per-node count updates + // happen in executeNodes() after each node completes. + scheduler.scheduleAtFixedRate(() -> { + try { + AutomationDatabaseUtility.touchHeartbeat(runId); + } catch (Exception e) { + classLogger.warn("Heartbeat update failed for run {}: {}", runId, e.getMessage()); + } + }, AutomationConstants.HEARTBEAT_INTERVAL_SECONDS, + AutomationConstants.HEARTBEAT_INTERVAL_SECONDS, TimeUnit.SECONDS); + return scheduler; + } + + // -- Cancellation Support ------------------------------------------------------ + + /** + * Requests cancellation of a running automation. Called by CancelAutomationRunReactor. + * Cancellation takes effect between nodes (cannot interrupt mid-pixel), or mid-wait for + * nodes that check the flag during blocking operations (e.g. WaitNodeExecutor). + */ + public static boolean requestCancellation(String runId) { + AtomicBoolean flag = CANCELLATION_FLAGS.get(runId); + if (flag != null) { + flag.set(true); + return true; + } + return false; + } + + /** + * Seeds the current (background executor) thread's ThreadStore with a snapshot of the + * caller's context. The reading getter forces lazy creation of this thread's map so the + * subsequent putAll has a target. Paired with {@code ThreadStore.remove()} in a finally + * block so pooled threads never leak context between runs. + */ + private static void installThreadContext(Map snapshot) { + if (snapshot == null || snapshot.isEmpty()) { + return; + } + ThreadStore.getInsightId(); // force creation of this thread's ThreadStore map + ThreadStore.setThreadMapObject(snapshot); + } + + // (resolve, getNodeTimeout, applyOutputTransform, strCfg, coerceToMap, loadAutomationDoc, + // topoSort moved to AutomationExecutionUtils) + + // -- Helpers ------------------------------------------------------------------- + + private String getProjectId() { + String projectId = this.keyValue.get(this.keysToGet[0]); + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + return projectId; + } + + private String getUserId() { + if (this.insight.getUser() != null && this.insight.getUser().getPrimaryLoginToken() != null) { + return this.insight.getUser().getPrimaryLoginToken().getId(); + } + return "system"; + } + + private String determineTriggerType(String resumeRunId) { + if (resumeRunId != null && !resumeRunId.isEmpty()) { + return AutomationConstants.TRIGGER_RESUME; + } + // Explicit triggerType param (webhook, storage-poll, db-poll) takes precedence + String explicit = this.keyValue.get(this.keysToGet[3]); + if (explicit != null && !explicit.isBlank()) { + return explicit.toUpperCase().replace("-", "_"); + } + String manual = this.keyValue.get(this.keysToGet[1]); + if ("true".equalsIgnoreCase(manual)) { + return AutomationConstants.TRIGGER_MANUAL; + } + return AutomationConstants.TRIGGER_SCHEDULED; + } + + private Map buildInitialScope(String runId) { + Map scope = new HashMap<>(); + String now = Instant.now().toString(); + scope.put("date", now.substring(0, 10)); + scope.put("triggered_at", now); + scope.put("run_id", runId); + return scope; + } + + private Timestamp toTimestamp(Instant instant) { + return Utility.getSqlTimestampUTC( + LocalDateTime.ofInstant(instant, ZoneOffset.UTC)); + } + + // -- Result Building ----------------------------------------------------------- + + private Map buildNodeResult(String nodeId, String nodeLabel, + String status, long durationMs, String outputPreview, String errorMessage) { + Map result = new HashMap<>(); + result.put(AutomationConstants.NODE_ID, nodeId); + result.put(AutomationConstants.NODE_LABEL, nodeLabel); + result.put(AutomationConstants.STATUS, status); + result.put(AutomationConstants.DURATION_MS, durationMs); + if (outputPreview != null) { + result.put(AutomationConstants.OUTPUT_PREVIEW, outputPreview); + } + if (errorMessage != null) { + result.put(AutomationConstants.ERROR_MESSAGE, errorMessage); + } + return result; + } + + private Map buildRunResult(String runId, String projectId, String status, + int totalNodes, int completedNodes, String failedNodeId, + List> nodeResults) { + // Read actual timestamps from DB rather than synthesizing "now" on every call. + Map stored = AutomationDatabaseUtility.getRunDetail(runId); + Map result = new HashMap<>(); + result.put(AutomationConstants.RUN_ID, runId); + result.put(AutomationConstants.PROJECT_ID, projectId); + result.put(AutomationConstants.STATUS, status); + result.put(AutomationConstants.TOTAL_NODES, totalNodes); + result.put(AutomationConstants.COMPLETED_NODES, completedNodes); + if (stored != null) { + result.put(AutomationConstants.STARTED_AT, stored.get(AutomationConstants.STARTED_AT)); + result.put(AutomationConstants.COMPLETED_AT, stored.get(AutomationConstants.COMPLETED_AT)); + } + if (failedNodeId != null) { + result.put(AutomationConstants.FAILED_NODE_ID, failedNodeId); + } + result.put("nodeResults", nodeResults); + return result; + } +} diff --git a/src/prerna/reactor/automation/nodes/AutomationNodeContext.java b/src/prerna/reactor/automation/nodes/AutomationNodeContext.java new file mode 100644 index 00000000000..f76d62ef44e --- /dev/null +++ b/src/prerna/reactor/automation/nodes/AutomationNodeContext.java @@ -0,0 +1,125 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import prerna.om.Insight; + +/** + * Single param object bundling everything an {@link IAutomationNodeExecutor} needs to run one + * node - replacing the inconsistent, differently-shaped per-method argument lists the previous + * {@code executeXNode} private methods each had (e.g. {@code executeWaitNode(node, scope, + * configMap)} took 3 args, {@code executeSwitchNode(runId, node, scope, configMap, + * ancestorProjectIds)} took 5). + * + *

Immutable - {@code scope} and {@code ancestorProjectIds} are references to the caller's + * live, mutable collections (executors write node outputs back into {@code scope} exactly as + * the previous {@code executeXNode} methods did), but the context object itself carries no + * mutable state of its own. + * + * @param runId the current run's id + * @param projectId the project id this run belongs to + * @param node this node's full definition from {@code automation.json} + * ({@code id}, {@code label}, {@code type}, {@code config}, ...) + * @param scope the run's current execution scope (prior node outputs, keyed by + * {@code outputVar}) - mutable, shared with the caller + * @param configMap the automation's {@code automation-config.json} key/value pairs + * @param ancestorProjectIds the chain of project ids already executing on this call stack, + * including this run's own project id - used by + * {@code SubAutomationNodeExecutor} for the self/transitive-call cycle + * guard + * @param insight the execution context - engines/reactors this node calls need it + * @param cancelFlag the run's cancellation flag, already resolved once by the caller + * (cluster-safe: reflects both the local {@code AtomicBoolean} and + * the DB {@code CANCEL_REQUESTED} column at the time this node + * started) - executors that loop internally (e.g. a future retry/ + * backoff inside a single node) should check this between iterations + * @param nodeDispatcher recurses into a single inner/branch node - see {@link NodeDispatcher}. + * Only used by composite executors (conditional, while-loop, try-catch, + * switch, retry, parallel); {@code null} is never passed - executors + * that don't need it simply don't call it + * @param childAutomationRunner runs an entire target project's automation graph as a nested child + * run - see {@link ChildAutomationRunner}. Only used by + * {@code SubAutomationNodeExecutor} + */ +public record AutomationNodeContext( + String runId, + String projectId, + Map node, + Map scope, + Map configMap, + Set ancestorProjectIds, + Insight insight, + AtomicBoolean cancelFlag, + NodeDispatcher nodeDispatcher, + ChildAutomationRunner childAutomationRunner) { + + /** Convenience accessor for this node's {@code id} field. */ + public String nodeId() { + return (String) node.get("id"); + } + + /** Convenience accessor for this node's {@code label} field. */ + public String nodeLabel() { + Object label = node.get("label"); + return label != null ? label.toString() : "unnamed"; + } + + /** Convenience accessor for this node's {@code type} field. */ + public String nodeType() { + return (String) node.get("type"); + } + + /** Convenience accessor for this node's {@code config} field, or an empty map if absent. */ + @SuppressWarnings("unchecked") + public Map config() { + Object config = node.get("config"); + return config instanceof Map ? (Map) config : Map.of(); + } + + /** + * Same list-of-strings shape used throughout the previous {@code executeXNode} methods for + * a branch/loop/case's inner nodes (e.g. {@code trueGraph.nodes}, {@code subGraph.nodes}). + */ + @SuppressWarnings("unchecked") + public static List> graphNodes(Map graph) { + return graph != null && graph.get("nodes") instanceof List + ? (List>) graph.get("nodes") : null; + } + + /** See {@link #graphNodes(Map)} - the corresponding {@code edges} list. */ + @SuppressWarnings("unchecked") + public static List> graphEdges(Map graph) { + return graph != null && graph.get("edges") instanceof List + ? (List>) graph.get("edges") : null; + } +} diff --git a/src/prerna/reactor/automation/nodes/ChildAutomationRunner.java b/src/prerna/reactor/automation/nodes/ChildAutomationRunner.java new file mode 100644 index 00000000000..6ab78ebbe26 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/ChildAutomationRunner.java @@ -0,0 +1,70 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Callback bound to {@code TriggerAutomationReactor.executeNodes}, used by + * {@code SubAutomationNodeExecutor} to run a target project's entire automation graph to completion + * as a nested, synchronous child run - distinct from {@link NodeDispatcher}, which runs a single + * inner node. + * + *

This is real orchestration (its own cancellation flag, its own heartbeat, its own DB run + * row/active-run claim) and intentionally stays owned by {@code TriggerAutomationReactor} rather + * than being reimplemented by the node-executor layer - see the "what does NOT change" scoping + * note on ticket #2746. + */ +@FunctionalInterface +public interface ChildAutomationRunner { + + /** + * Runs an already-claimed, already-topo-sorted child automation run to completion and returns + * its final run result (the same shape {@code buildRunResult} produces for a top-level run). + * + * @param childRunId the child run's id (already inserted into {@code AUTOMATION_RUNS} + * and already holding the active-run claim for {@code targetProjectId}) + * @param targetProjectId the project id whose automation is being run + * @param orderedNodes the child automation's nodes, already topologically sorted + * @param configMap the child project's {@code automation-config.json} key/value pairs + * @param priorOutputs prior node outputs to skip on resume - empty for a fresh + * sub-automation call, never itself resumable independently + * @param extraInitialScope the resolved {@code inputMapping} values to seed into the + * child's initial scope + * @param ancestorProjectIds the chain of project ids already executing on this call stack, + * including {@code targetProjectId} - used for the + * self/transitive-call cycle guard on any further nested calls + * @return the child run's final result map, containing at minimum {@code STATUS} + */ + Map run(String childRunId, String targetProjectId, + List> orderedNodes, Map configMap, + Map priorOutputs, Map extraInitialScope, + Set ancestorProjectIds); +} diff --git a/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java new file mode 100644 index 00000000000..033f769d918 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java @@ -0,0 +1,72 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import prerna.reactor.automation.AutomationExecutionUtils; +import prerna.reactor.automation.PixelExecutionUtils; + +import java.util.Map; + +/** + * Executes a "database-engine" node: builds and runs a {@code SqlQuery(...)} Pixel call from + * structured {@code config} on the backend, instead of trusting a frontend-precompiled + * {@code builtPixel} string (ticket #2743). Reuses {@code SqlQueryReactor}/ + * {@code AbstractSqlQueryReactor} unmodified via the normal Pixel path - including its existing + * SELECT-vs-mutation permission split ({@code userCanViewEngine} for reads, + * {@code userCanEditEngine} for writes) - so no security logic is duplicated here. + * + *

Config: {@code {engineId, operation: "read"|"write", expression (the SQL), limit, commit}}. + */ +public final class DatabaseEngineNodeExecutor implements IAutomationNodeExecutor { + + @Override + public Object execute(AutomationNodeContext ctx) { + Map config = ctx.config(); + String nodeLabel = ctx.nodeLabel(); + + String engineId = EngineNodeSupport.required(config, "engineId", "Database-engine", nodeLabel); + String sql = EngineNodeSupport.required(config, "expression", "Database-engine", nodeLabel); + String operation = EngineNodeSupport.optional(config, "operation", "read"); + + Map scope = ctx.scope(); + Map configMap = ctx.configMap(); + String encodedEngineId = EngineNodeSupport.resolveEncoded(engineId, scope, configMap); + String encodedSql = EngineNodeSupport.resolveEncoded(sql, scope, configMap); + + String pixel; + if ("write".equals(operation)) { + pixel = "SqlQuery(database=[" + encodedEngineId + "], query=[" + encodedSql + "], commit=[true]);"; + } else { + int limit = EngineNodeSupport.optionalInt(config, "limit", 50); + pixel = "SqlQuery(database=[" + encodedEngineId + "], query=[" + encodedSql + "], limit=[" + limit + "]);"; + } + + int timeoutSeconds = AutomationExecutionUtils.getNodeTimeout(ctx.node()); + return PixelExecutionUtils.runAndCollect(ctx.insight(), pixel, timeoutSeconds); + } +} diff --git a/src/prerna/reactor/automation/nodes/EngineNodeSupport.java b/src/prerna/reactor/automation/nodes/EngineNodeSupport.java new file mode 100644 index 00000000000..0704d5e0557 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/EngineNodeSupport.java @@ -0,0 +1,172 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; +import com.google.gson.JsonSyntaxException; + +import prerna.reactor.automation.AutomationExecutionUtils; + +/** + * Shared helpers for the 5 engine-type node executors (database/model/vector/storage/function- + * engine). Each of those builds a validated Pixel call from structured {@code node.config} on + * the backend - reusing the target reactor's existing security/validation/query-building logic + * unmodified by running it through the normal Pixel path - rather than trusting a frontend- + * precompiled {@code builtPixel} string (see ticket #2743). + * + *

Every templated/dynamic value is wrapped in {@code ...}, not just the + * "obviously free text" fields (command/query/prompt) that the frontend's preview-only + * equivalent ({@code buildPixelPreview()} in {@code automation-utils.ts}) encodes. Any config field + * can carry a {@code ${var}} reference to an upstream node's output - which may itself be + * LLM-generated or attacker-influenced content - so any field that isn't a fixed structural + * literal (an engine id selected from a dropdown is still encoded defensively) gets the same + * injection protection. This is deliberately more conservative than the frontend preview + * builder - see epic sub-issue #2750 (verifying encode-wrapping coverage) for the general + * concern this addresses. + * + *

The exception is fields that must be embedded as a raw, unquoted Pixel map/list literal + * (e.g. {@code paramValues}, {@code metadata}) because the target reactor's key expects an + * actual parsed Map/List noun, not a String - {@code } would change how the parser + * types the literal. Those fields go through {@link #resolveAndValidateJsonLiteral} instead: + * {@code ${var}} substitution happens first (on the raw field, not the whole assembled pixel + * string), then the result is validated as syntactically complete, balanced JSON before it is + * spliced in unquoted. This is a real defense, not just documentation of the gap - a resolved + * value that doesn't parse as balanced JSON (e.g. one containing {@code "], SomeReactor(x=["}) + * is rejected outright rather than silently embedded, so it cannot break out of the literal's + * boundaries and inject arbitrary Pixel syntax. + */ +final class EngineNodeSupport { + + private EngineNodeSupport() { + // static utility - no instantiation + } + + /** Wraps a value as an {@code }-protected Pixel string literal, e.g. {@code "foo"}. */ + static String encoded(Object value) { + return "\"" + (value != null ? value : "") + "\""; + } + + /** + * Resolves {@code ${var}} references in a raw config value, then wraps the already-resolved + * result in {@code ...}. Every engine-type executor resolves each field + * individually this way and builds its Pixel call from already-resolved pieces - matching + * the established convention elsewhere in this package (e.g. {@code EmailNodeExecutor}) - + * rather than assembling a still-templated pixel string and resolving it as one final pass. + * Resolving per-field first, and never resolving the assembled string a second time, avoids + * a subtle double-substitution risk: if a second whole-string resolve pass ran after this + * value was already embedded, and the resolved content happened to itself contain a literal + * {@code ${...}} sequence matching a scope key (e.g. upstream data that isn't a template but + * looks like one), it would get incorrectly re-substituted. + */ + static String resolveEncoded(String rawTemplate, Map scope, Map configMap) { + return encoded(AutomationExecutionUtils.resolve(rawTemplate, scope, configMap)); + } + + /** + * Resolves {@code ${var}} references in a raw config value that will be spliced into a + * Pixel map/list literal position unquoted (e.g. {@code paramValues=[]}), + * then validates the resolved text is syntactically complete, balanced JSON (an object or + * array). Throws rather than returning unvalidated content, since a value that isn't + * well-formed, self-contained JSON could contain a sequence that breaks out of the literal + * and injects arbitrary Pixel syntax once embedded into the assembled pixel string. + */ + static String resolveAndValidateJsonLiteral(String rawTemplate, Map scope, + Map configMap, String fieldName, String nodeTypeLabel, String nodeLabel) { + String resolved = AutomationExecutionUtils.resolve(rawTemplate, scope, configMap); + try { + JsonElement el = JsonParser.parseString(resolved); + if (!el.isJsonObject() && !el.isJsonArray()) { + throw new IllegalArgumentException("must be a JSON object or array, got: " + resolved); + } + } catch (JsonSyntaxException | IllegalArgumentException e) { + throw new IllegalArgumentException(nodeTypeLabel + " node \"" + nodeLabel + "\": '" + fieldName + + "' did not resolve to valid, complete JSON after substituting ${var} references (" + + e.getMessage() + ") - refusing to embed unvalidated content into the Pixel call", e); + } + return resolved; + } + + /** + * Resolves {@code ${var}} references in a raw config value that will be embedded as a + * quoted, quote-escaped Pixel string (e.g. {@code map=[""]}, used by + * {@code FunctionEngineNodeExecutor}), then escapes the resolved text for that position. + * Same ordering rationale as {@link #resolveAndValidateJsonLiteral}: resolve the field alone + * first, escape the actual resolved content, then splice - not escape-then-resolve, which + * would let a substituted value's own quotes reach the pixel string unescaped and break out + * of the surrounding {@code "..."} boundary. + */ + static String resolveAndEscapeForQuotedPixelString(String rawTemplate, Map scope, + Map configMap) { + String resolved = AutomationExecutionUtils.resolve(rawTemplate, scope, configMap); + return resolved.replace("\\", "\\\\").replace("\"", "\\\""); + } + + /** + * Reads a required config field as a String, throwing a clear, node-labeled error if it's + * missing or blank - backend-side re-validation, since an automation.json can be edited + * directly (API call bypassing the FE, or a future FE bug) and must never be trusted implicitly. + */ + static String required(Map config, String key, String nodeTypeLabel, String nodeLabel) { + Object v = config.get(key); + if (v == null || v.toString().isBlank()) { + throw new IllegalArgumentException(nodeTypeLabel + " node \"" + nodeLabel + "\": '" + key + "' is required"); + } + return v.toString(); + } + + /** Reads an optional config field as a String, or {@code def} if missing/blank. */ + static String optional(Map config, String key, String def) { + Object v = config.get(key); + return (v == null || v.toString().isBlank()) ? def : v.toString(); + } + + /** Reads an optional config field as a String, or {@code null} if missing/blank. */ + static String optional(Map config, String key) { + return optional(config, key, null); + } + + /** Reads an optional config field as an int, or {@code def} if missing/blank/unparseable. */ + static int optionalInt(Map config, String key, int def) { + Object v = config.get(key); + if (v == null) return def; + try { + return Integer.parseInt(v.toString().trim()); + } catch (NumberFormatException e) { + return def; + } + } + + /** Ensures a Pixel statement string ends with a semicolon, as the parser requires. */ + static String terminated(String pixel) { + String trimmed = pixel.trim(); + return trimmed.endsWith(";") ? trimmed : trimmed + ";"; + } +} diff --git a/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java new file mode 100644 index 00000000000..5fbaf29b4cd --- /dev/null +++ b/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java @@ -0,0 +1,73 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +import prerna.reactor.automation.AutomationExecutionUtils; +import prerna.reactor.automation.PixelExecutionUtils; + +/** + * Executes a "function-engine" node: builds and runs an {@code ExecuteFunctionEngine(...)} / + * {@code ExecuteStreamingFunctionEngine(...)} Pixel call from structured {@code config} on the + * backend, instead of trusting a frontend-precompiled {@code builtPixel} string (ticket #2743). + * Reuses the existing {@code ExecuteFunctionEngineReactor}/{@code ExecuteStreamingFunctionEngineReactor} + * unmodified via the normal Pixel path. + * + *

Config: {@code {engineId, operation: "default"|"streaming", params (a JSON object string)}}. + * + *

{@code params} is passed as a quoted, quote-escaped string (matching the frontend's existing + * {@code buildPixelPreview()} shape exactly, rather than {@code }-wrapped like other + * fields here) because the target reactor's {@code map} key expects a Pixel Map noun, and + * {@code } would change how the parser types the literal. {@code ${var}} substitution + * happens on the raw field first (via + * {@link EngineNodeSupport#resolveAndEscapeForQuotedPixelString}), then the resolved text is + * quote-escaped before it is embedded - so a substituted value's own quotes can't break out of + * the surrounding {@code "..."} boundary. + */ +public final class FunctionEngineNodeExecutor implements IAutomationNodeExecutor { + + @Override + public Object execute(AutomationNodeContext ctx) { + Map config = ctx.config(); + String nodeLabel = ctx.nodeLabel(); + Map scope = ctx.scope(); + Map configMap = ctx.configMap(); + String engineId = EngineNodeSupport.required(config, "engineId", "Function-engine", nodeLabel); + String operation = EngineNodeSupport.optional(config, "operation"); + String params = EngineNodeSupport.optional(config, "params", "{}"); + String escapedParams = EngineNodeSupport.resolveAndEscapeForQuotedPixelString(params, scope, configMap); + + String command = "streaming".equals(operation) ? "ExecuteStreamingFunctionEngine" : "ExecuteFunctionEngine"; + String pixel = command + "(engine=[" + EngineNodeSupport.resolveEncoded(engineId, scope, configMap) + + "], map=[\"" + escapedParams + "\"]);"; + + int timeoutSeconds = AutomationExecutionUtils.getNodeTimeout(ctx.node()); + return PixelExecutionUtils.runAndCollect(ctx.insight(), pixel, timeoutSeconds); + } +} diff --git a/src/prerna/reactor/automation/nodes/IAutomationNodeExecutor.java b/src/prerna/reactor/automation/nodes/IAutomationNodeExecutor.java new file mode 100644 index 00000000000..8ee9645b86d --- /dev/null +++ b/src/prerna/reactor/automation/nodes/IAutomationNodeExecutor.java @@ -0,0 +1,65 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +/** + * Executes one automation node's operation, given all the context that node type needs. + * + *

One concrete implementation per {@code node.type} value (e.g. {@code WaitNodeExecutor}, + * {@code ConditionalNodeExecutor}, {@code DatabaseEngineNodeExecutor}), resolved via a + * {@code Map} registry in + * {@link prerna.reactor.automation.TriggerAutomationReactor#executeSingleNode} instead of the + * previous {@code if/else} chain keyed on {@code type}. + * + *

Mirrors the shape already established in this codebase for "one conceptual operation, many + * type-specific implementations, resolved by a type key" - see + * {@link prerna.engine.api.IModelEngine} (resolved via {@code Utility.getModel(engineId)}) and + * {@link prerna.engine.api.IMCP}. + * + *

Implementations should be stateless and safe to share as a single static instance across + * concurrent runs - all per-run/per-node state is passed in via {@link AutomationNodeContext}, not + * held on the executor instance. + */ +public interface IAutomationNodeExecutor { + + /** + * Executes this node's operation and returns its raw output - the same shape previously + * returned by each {@code executeXNode} method (a String, a JSON string, or a + * {@code Map} that the caller will serialize). The caller + * ({@code executeSingleNode}) is responsible for applying the node's output transform, + * generating the preview, and checkpointing success/failure to the database - executors + * should not do any of that themselves. + * + * @param ctx the node's execution context - node definition, scope, config, and callbacks + * for recursing into sibling/child nodes where the node type requires it + * @return the node's raw output + * @throws Exception on any failure - the caller catches and records it as a failed node, + * following the same contract the previous {@code executeXNode} methods had + */ + Object execute(AutomationNodeContext ctx) throws Exception; +} diff --git a/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java new file mode 100644 index 00000000000..471ef1e27b4 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java @@ -0,0 +1,112 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +import prerna.reactor.automation.AutomationExecutionUtils; +import prerna.reactor.automation.PixelExecutionUtils; + +/** + * Executes a "model-engine" node: builds and runs the matching model-operation Pixel call + * ({@code LLM}/{@code Embeddings}/{@code Vision}/{@code NER}) from structured {@code config} on + * the backend, instead of trusting a frontend-precompiled {@code builtPixel} string (ticket + * #2743). Reuses the existing {@code LLMReactor}/{@code EmbeddingsReactor}/{@code VisionReactor}/ + * {@code NERReactor} unmodified via the normal Pixel path. + * + *

Config: {@code {engineId, operation: "llm"|"embeddings"|"vision"|"ner", command, context, + * paramValues, values, image, prompt, entities}}. + */ +public final class ModelEngineNodeExecutor implements IAutomationNodeExecutor { + + @Override + public Object execute(AutomationNodeContext ctx) { + Map config = ctx.config(); + String nodeLabel = ctx.nodeLabel(); + Map scope = ctx.scope(); + Map configMap = ctx.configMap(); + String engineId = EngineNodeSupport.required(config, "engineId", "Model-engine", nodeLabel); + String operation = EngineNodeSupport.optional(config, "operation", "llm"); + String encodedEngineId = EngineNodeSupport.resolveEncoded(engineId, scope, configMap); + + String pixel; + switch (operation) { + case "embeddings": { + String values = EngineNodeSupport.required(config, "values", "Model-engine", nodeLabel); + pixel = "Embeddings(engine=[" + encodedEngineId + + "], values=[" + EngineNodeSupport.resolveEncoded(values, scope, configMap) + "]);"; + break; + } + case "vision": { + String command = EngineNodeSupport.required(config, "command", "Model-engine", nodeLabel); + String image = EngineNodeSupport.required(config, "image", "Model-engine", nodeLabel); + pixel = "Vision(engine=[" + encodedEngineId + + "], command=[" + EngineNodeSupport.resolveEncoded(command, scope, configMap) + + "], image=[" + EngineNodeSupport.resolveEncoded(image, scope, configMap) + "]);"; + break; + } + case "ner": { + String prompt = EngineNodeSupport.required(config, "prompt", "Model-engine", nodeLabel); + String entities = EngineNodeSupport.required(config, "entities", "Model-engine", nodeLabel); + pixel = "NER(engine=[" + encodedEngineId + + "], prompt=[" + EngineNodeSupport.resolveEncoded(prompt, scope, configMap) + + "], entities=[" + EngineNodeSupport.resolveEncoded(entities, scope, configMap) + "]);"; + break; + } + default: { + // llm + String command = EngineNodeSupport.required(config, "command", "Model-engine", nodeLabel); + StringBuilder pixelBuilder = new StringBuilder("LLM(engine=[") + .append(encodedEngineId) + .append("], command=[").append(EngineNodeSupport.resolveEncoded(command, scope, configMap)).append("]"); + String context = EngineNodeSupport.optional(config, "context"); + if (context != null) { + pixelBuilder.append(", context=[").append(EngineNodeSupport.resolveEncoded(context, scope, configMap)).append("]"); + } + String paramValues = EngineNodeSupport.optional(config, "paramValues"); + if (paramValues != null) { + // paramValues is a Pixel map literal (e.g. {"key":"value"}), not a string - + // ReactorKeysEnum.PARAM_VALUES_MAP expects an actual map, so this must stay + // unquoted/un-encoded to match the FE's existing wire contract (buildPixelPreview + // emits it the same way). resolveAndValidateJsonLiteral substitutes ${var} + // refs on this field alone and rejects anything that doesn't resolve to + // complete, balanced JSON, so a value can't break out of the map literal and + // inject arbitrary Pixel syntax. + String resolvedParamValues = EngineNodeSupport.resolveAndValidateJsonLiteral( + paramValues, scope, configMap, "paramValues", "Model-engine", nodeLabel); + pixelBuilder.append(", paramValues=[").append(resolvedParamValues).append("]"); + } + pixelBuilder.append(");"); + pixel = pixelBuilder.toString(); + } + } + + int timeoutSeconds = AutomationExecutionUtils.getNodeTimeout(ctx.node()); + return PixelExecutionUtils.runAndCollect(ctx.insight(), pixel, timeoutSeconds); + } +} diff --git a/src/prerna/reactor/automation/nodes/NodeDispatcher.java b/src/prerna/reactor/automation/nodes/NodeDispatcher.java new file mode 100644 index 00000000000..cf9c7e1135c --- /dev/null +++ b/src/prerna/reactor/automation/nodes/NodeDispatcher.java @@ -0,0 +1,60 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +/** + * Callback bound to {@code TriggerAutomationReactor.executeSingleNode}, allowing composite node + * executors (e.g. {@code ConditionalNodeExecutor}, {@code WhileLoopNodeExecutor}, + * {@code TryCatchNodeExecutor}, {@code SwitchNodeExecutor}, {@code RetryNodeExecutor}, + * {@code ParallelNodeExecutor}) to recurse into a branch/loop/case's inner nodes without + * depending on {@code TriggerAutomationReactor} directly. + * + *

{@code node}/{@code scope} are the only two arguments that vary per recursive call - the + * rest of the calling node's context ({@code runId}, {@code configMap}, {@code ancestorProjectIds}) + * stays fixed across the recursion and is captured by the lambda this is bound to. + */ +@FunctionalInterface +public interface NodeDispatcher { + + /** + * Executes a single inner node exactly as {@code executeSingleNode} would for a top-level + * node - markNodeRunning, timing, output-transform, preview, checkpointing, and + * success/failure result building all happen inside this call, matching the contract + * every branch of the original {@code if/else} chain relied on implicitly. + * + * @param node the inner node definition (from a {@code trueGraph}/{@code falseGraph}/ + * {@code subGraph}/{@code tryGraph}/{@code catchGraph}/case branch) + * @param scope the current execution scope - inner nodes read prior outputs from it and + * (via the caller) may write their own output back into it + * @return the same node-result map shape {@code executeSingleNode} normally returns - + * contains at minimum {@code STATUS}, and on success {@code outputValue} + */ + Map dispatch(Map node, Map scope); +} diff --git a/src/prerna/reactor/automation/nodes/PixelNodeExecutor.java b/src/prerna/reactor/automation/nodes/PixelNodeExecutor.java new file mode 100644 index 00000000000..f7802903594 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/PixelNodeExecutor.java @@ -0,0 +1,101 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +import prerna.reactor.automation.AutomationConstants; +import prerna.reactor.automation.AutomationExecutionUtils; +import prerna.reactor.automation.PixelExecutionUtils; + +/** + * Default/fallback executor - runs a node's frontend-precompiled {@code builtPixel} verbatim + * (after {@code ${var}} substitution). This is the correct behavior for node types that are + * genuinely arbitrary or composed Pixel with no single backing engine: + *

    + *
  • {@code trigger} - returns the run's trigger timestamp, no Pixel execution
    • + *
  • {@code app} - runs an arbitrary multi-engine recipe scoped to a project + * (e.g. {@code IndexPubmedDocuments(database=[..], storage=[..], vector=[..], ...)})
    • + *
  • {@code custom-pixel} - arbitrary user-authored Pixel, optionally scoped to an app via + * a leading {@code LoadApp(...)} setup call
    • + *
+ * + *

Unlike the previous {@code executeNodePixel}, this is not used for + * {@code database-engine}/{@code model-engine}/{@code vector-engine}/{@code storage-engine}/ + * {@code function-engine} nodes - those have their own dedicated executors that read structured + * {@code config} and call the matching engine/reactor directly, rather than trusting a + * frontend-precompiled Pixel string (see ticket #2743). + */ +public final class PixelNodeExecutor implements IAutomationNodeExecutor { + + @Override + @SuppressWarnings("unchecked") + public Object execute(AutomationNodeContext ctx) { + Map node = ctx.node(); + Map scope = ctx.scope(); + Map configMap = ctx.configMap(); + String type = ctx.nodeType(); + + if (AutomationConstants.NODE_TRIGGER.equals(type)) { + return scope.get("triggered_at"); + } + + String builtPixel = (String) node.get("builtPixel"); + if (builtPixel == null || builtPixel.isBlank() || builtPixel.startsWith("//")) { + throw new IllegalStateException("Node \"" + node.get("label") + + "\" has no compiled pixel - please Save the automation before running"); + } + + int timeoutSeconds = AutomationExecutionUtils.getNodeTimeout(node); + String resolvedPixel = AutomationExecutionUtils.resolve(builtPixel, scope, configMap); + + // For custom-pixel nodes with an appId, the builtPixel is "LoadApp(...); actualPixel". + // Run LoadApp as a fire-and-forget setup step so only the actual pixel's output + // is captured and stored as the node's result. + if (AutomationConstants.NODE_CUSTOM_PIXEL.equals(type)) { + Map config = (Map) node.get("config"); + Object appIdObj = config != null ? config.get("appId") : null; + if (appIdObj != null && !appIdObj.toString().isBlank()) { + int semicolon = resolvedPixel.indexOf(';'); + if (semicolon > 0) { + String setupPixel = resolvedPixel.substring(0, semicolon).trim(); + String actualPixel = resolvedPixel.substring(semicolon + 1).trim(); + if (!setupPixel.isBlank() && !actualPixel.isBlank()) { + // SEMOSS pixel parser requires a trailing semicolon on every statement + if (!setupPixel.endsWith(";")) setupPixel += ";"; + if (!actualPixel.endsWith(";")) actualPixel += ";"; + ctx.insight().runPixel(setupPixel); // set context, discard output + return PixelExecutionUtils.runAndCollect(ctx.insight(), actualPixel, timeoutSeconds); + } + } + } + } + + return PixelExecutionUtils.runAndCollect(ctx.insight(), resolvedPixel, timeoutSeconds); + } +} diff --git a/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java new file mode 100644 index 00000000000..9a2b9658c29 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java @@ -0,0 +1,110 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +import prerna.reactor.automation.AutomationExecutionUtils; +import prerna.reactor.automation.PixelExecutionUtils; + +/** + * Executes a "storage-engine" node: builds and runs the matching storage-operation Pixel call + * from structured {@code config} on the backend, instead of trusting a frontend-precompiled + * {@code builtPixel} string (ticket #2743). Reuses the existing + * {@code ListStoragePathReactor}/{@code PullFromStorageReactor}/{@code PushToStorageReactor}/ + * {@code DeleteFromStorageReactor}/{@code GetStorageFileAsBase64Reactor} unmodified via the + * normal Pixel path. + * + *

Config: {@code {engineId, operation: "list"|"download"|"upload"|"delete"|"read-base64", + * storagePath, filePath, metadata}}. + */ +public final class StorageEngineNodeExecutor implements IAutomationNodeExecutor { + + @Override + public Object execute(AutomationNodeContext ctx) { + Map config = ctx.config(); + String nodeLabel = ctx.nodeLabel(); + Map scope = ctx.scope(); + Map configMap = ctx.configMap(); + String engineId = EngineNodeSupport.required(config, "engineId", "Storage-engine", nodeLabel); + String operation = EngineNodeSupport.optional(config, "operation", "list"); + String encodedEngineId = EngineNodeSupport.resolveEncoded(engineId, scope, configMap); + + String pixel; + switch (operation) { + case "download": { + String storagePath = EngineNodeSupport.required(config, "storagePath", "Storage-engine", nodeLabel); + String filePath = EngineNodeSupport.required(config, "filePath", "Storage-engine", nodeLabel); + pixel = "PullFromStorage(storage=[" + encodedEngineId + + "], storagePath=[" + EngineNodeSupport.resolveEncoded(storagePath, scope, configMap) + + "], filePath=[" + EngineNodeSupport.resolveEncoded(filePath, scope, configMap) + "]);"; + break; + } + case "upload": { + String storagePath = EngineNodeSupport.required(config, "storagePath", "Storage-engine", nodeLabel); + String filePath = EngineNodeSupport.required(config, "filePath", "Storage-engine", nodeLabel); + StringBuilder pixelBuilder = new StringBuilder("PushToStorage(storage=[") + .append(encodedEngineId) + .append("], storagePath=[").append(EngineNodeSupport.resolveEncoded(storagePath, scope, configMap)) + .append("], filePath=[").append(EngineNodeSupport.resolveEncoded(filePath, scope, configMap)).append("]"); + String metadata = EngineNodeSupport.optional(config, "metadata"); + if (metadata != null) { + // Pixel map literal, not a string - see ModelEngineNodeExecutor's paramValues + // handling for the same resolve-then-validate treatment. + String resolvedMetadata = EngineNodeSupport.resolveAndValidateJsonLiteral( + metadata, scope, configMap, "metadata", "Storage-engine", nodeLabel); + pixelBuilder.append(", metadata=[").append(resolvedMetadata).append("]"); + } + pixelBuilder.append(");"); + pixel = pixelBuilder.toString(); + break; + } + case "delete": { + String storagePath = EngineNodeSupport.required(config, "storagePath", "Storage-engine", nodeLabel); + pixel = "DeleteFromStorage(storage=[" + encodedEngineId + + "], storagePath=[" + EngineNodeSupport.resolveEncoded(storagePath, scope, configMap) + "]);"; + break; + } + case "read-base64": { + String storagePath = EngineNodeSupport.required(config, "storagePath", "Storage-engine", nodeLabel); + pixel = "GetStorageFileAsBase64(storage=[" + encodedEngineId + + "], storagePath=[" + EngineNodeSupport.resolveEncoded(storagePath, scope, configMap) + "]);"; + break; + } + default: { + // list + String storagePath = EngineNodeSupport.optional(config, "storagePath", "/"); + pixel = "ListStoragePath(storage=[" + encodedEngineId + + "], storagePath=[" + EngineNodeSupport.resolveEncoded(storagePath, scope, configMap) + "]);"; + } + } + + int timeoutSeconds = AutomationExecutionUtils.getNodeTimeout(ctx.node()); + return PixelExecutionUtils.runAndCollect(ctx.insight(), pixel, timeoutSeconds); + } +} diff --git a/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java new file mode 100644 index 00000000000..7c4ac012b95 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java @@ -0,0 +1,120 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +import prerna.reactor.automation.AutomationExecutionUtils; +import prerna.reactor.automation.PixelExecutionUtils; + +/** + * Executes a "vector-engine" node: builds and runs the matching vector-operation Pixel call from + * structured {@code config} on the backend, instead of trusting a frontend-precompiled + * {@code builtPixel} string (ticket #2743). Reuses the existing + * {@code VectorDatabaseQueryReactor}/{@code VectorAttachFileToSourceReactor}/ + * {@code CreateEmbeddingsFromVectorCSVFileReactor}/{@code ListDocumentsInVectorDatabaseReactor}/ + * {@code RemoveDocumentFromVectorDatabaseReactor}/{@code VectorFileDownloadReactor} unmodified via + * the normal Pixel path - including {@code CreateEmbeddingsFromVectorCSVFileReactor}'s + * substantial (~385 line) CSV-parsing/chunking logic, which this deliberately does not + * re-implement. + * + *

Config: {@code {engineId, operation: "search"|"add-file"|"add-csv"|"list"|"delete"| + * "download", command, limit, filePath, source, space, filePaths, paramValues, fileNames}}. + */ +public final class VectorEngineNodeExecutor implements IAutomationNodeExecutor { + + @Override + public Object execute(AutomationNodeContext ctx) { + Map config = ctx.config(); + String nodeLabel = ctx.nodeLabel(); + Map scope = ctx.scope(); + Map configMap = ctx.configMap(); + String engineId = EngineNodeSupport.required(config, "engineId", "Vector-engine", nodeLabel); + String operation = EngineNodeSupport.optional(config, "operation", "search"); + String encodedEngineId = EngineNodeSupport.resolveEncoded(engineId, scope, configMap); + + String pixel; + switch (operation) { + case "add-file": { + String filePath = EngineNodeSupport.required(config, "filePath", "Vector-engine", nodeLabel); + StringBuilder pixelBuilder = new StringBuilder("VectorAttachFileToSource(engine=[") + .append(encodedEngineId) + .append("], filePath=[").append(EngineNodeSupport.resolveEncoded(filePath, scope, configMap)).append("]"); + String source = EngineNodeSupport.optional(config, "source"); + if (source != null) pixelBuilder.append(", source=[").append(EngineNodeSupport.resolveEncoded(source, scope, configMap)).append("]"); + String space = EngineNodeSupport.optional(config, "space"); + if (space != null) pixelBuilder.append(", space=[").append(EngineNodeSupport.resolveEncoded(space, scope, configMap)).append("]"); + pixelBuilder.append(");"); + pixel = pixelBuilder.toString(); + break; + } + case "add-csv": { + String filePaths = EngineNodeSupport.required(config, "filePaths", "Vector-engine", nodeLabel); + StringBuilder pixelBuilder = new StringBuilder("CreateEmbeddingsFromVectorCSVFile(engine=[") + .append(encodedEngineId) + .append("], filePaths=[").append(EngineNodeSupport.resolveEncoded(filePaths, scope, configMap)).append("]"); + String paramValues = EngineNodeSupport.optional(config, "paramValues"); + if (paramValues != null) { + // Pixel map literal, not a string - see ModelEngineNodeExecutor's paramValues + // handling for the same resolve-then-validate treatment. + String resolvedParamValues = EngineNodeSupport.resolveAndValidateJsonLiteral( + paramValues, scope, configMap, "paramValues", "Vector-engine", nodeLabel); + pixelBuilder.append(", paramValues=[").append(resolvedParamValues).append("]"); + } + pixelBuilder.append(");"); + pixel = pixelBuilder.toString(); + break; + } + case "list": + pixel = "ListDocumentsInVectorDatabase(engine=[" + encodedEngineId + "]);"; + break; + case "delete": { + String fileNames = EngineNodeSupport.required(config, "fileNames", "Vector-engine", nodeLabel); + pixel = "RemoveDocumentFromVectorDatabase(engine=[" + encodedEngineId + + "], fileNames=[" + EngineNodeSupport.resolveEncoded(fileNames, scope, configMap) + "]);"; + break; + } + case "download": { + String fileNames = EngineNodeSupport.required(config, "fileNames", "Vector-engine", nodeLabel); + pixel = "VectorFileDownload(engine=[" + encodedEngineId + + "], fileNames=[" + EngineNodeSupport.resolveEncoded(fileNames, scope, configMap) + "]);"; + break; + } + default: { + // search + String command = EngineNodeSupport.required(config, "command", "Vector-engine", nodeLabel); + int limit = EngineNodeSupport.optionalInt(config, "limit", 5); + pixel = "VectorDatabaseQuery(engine=[" + encodedEngineId + + "], command=[" + EngineNodeSupport.resolveEncoded(command, scope, configMap) + "], limit=[" + limit + "]);"; + } + } + + int timeoutSeconds = AutomationExecutionUtils.getNodeTimeout(ctx.node()); + return PixelExecutionUtils.runAndCollect(ctx.insight(), pixel, timeoutSeconds); + } +} diff --git a/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java new file mode 100644 index 00000000000..636a068a2be --- /dev/null +++ b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java @@ -0,0 +1,90 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import prerna.reactor.automation.AutomationCancelledException; +import prerna.reactor.automation.AutomationDatabaseUtility; +import prerna.reactor.automation.AutomationExecutionUtils; + +/** + * Executes a "wait" node: sleeps for the configured number of seconds. + * The {@code seconds} value supports {@code ${var}} template substitution. + * Maximum 3600 seconds (1 hour) per invocation. + * + *

Sleeps in 5-second chunks, checking the cancellation flag between each chunk. + * This fixes the bug where a single {@code Thread.sleep(N)} call would block for the full + * duration even after a cancel request was received - the cancel check was only between + * nodes, so a long wait could not be interrupted until it completed naturally. + */ +public final class WaitNodeExecutor implements IAutomationNodeExecutor { + + private static final int CANCEL_CHECK_INTERVAL_SECONDS = 5; + + @Override + @SuppressWarnings("unchecked") + public Object execute(AutomationNodeContext ctx) { + Map node = ctx.node(); + Map config = (Map) node.get("config"); + String nodeLabel = ctx.nodeLabel(); + + String secondsTemplate = config.get("seconds") != null + ? config.get("seconds").toString() : "1"; + String resolved = AutomationExecutionUtils.resolve(secondsTemplate, ctx.scope(), ctx.configMap()); + + int seconds; + try { + seconds = Integer.parseInt(resolved.trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Wait node \"" + nodeLabel + + "\" - seconds value is not a valid integer after resolution: \"" + resolved + "\""); + } + seconds = Math.min(Math.max(seconds, 0), 3600); + + // Sleep in chunks so cancel requests are honored mid-wait rather than waiting + // for the full duration to complete. + int remaining = seconds; + while (remaining > 0) { + if (ctx.cancelFlag().get() || AutomationDatabaseUtility.isCancelRequested(ctx.runId())) { + throw new AutomationCancelledException("Wait node \"" + nodeLabel + "\" cancelled"); + } + int chunk = Math.min(remaining, CANCEL_CHECK_INTERVAL_SECONDS); + try { + TimeUnit.SECONDS.sleep(chunk); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Wait node \"" + nodeLabel + "\" was interrupted"); + } + remaining -= chunk; + } + + return seconds + " seconds"; + } +} diff --git a/src/prerna/reactor/scheduler/SchedulerConstants.java b/src/prerna/reactor/scheduler/SchedulerConstants.java index aaa3e5e4b4d..2c82484ca2c 100644 --- a/src/prerna/reactor/scheduler/SchedulerConstants.java +++ b/src/prerna/reactor/scheduler/SchedulerConstants.java @@ -128,6 +128,8 @@ private SchedulerConstants() { public static final String VARCHAR_250 = "VARCHAR (250)"; public static final String VARCHAR_255 = "VARCHAR (255)"; public static final String VARCHAR_512 = "VARCHAR (512)"; + public static final String VARCHAR_1000 = "VARCHAR (1000)"; + public static final String VARCHAR_2000 = "VARCHAR (2000)"; public static final String INTEGER = "INTEGER"; public static final String BOOLEAN = "BOOLEAN"; diff --git a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java index 9edd86100ea..458d864d813 100644 --- a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java +++ b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java @@ -124,6 +124,8 @@ import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_8; import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_80; import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_95; +import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_1000; +import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_2000; import java.util.ArrayList; import java.util.Arrays; @@ -132,6 +134,7 @@ import prerna.engine.impl.owl.AbstractOwlCreator; import prerna.engine.impl.owl.WriteOWLEngine; +import prerna.reactor.automation.AutomationConstants; public class SchedulerOwlCreator extends AbstractOwlCreator { @@ -275,6 +278,61 @@ public void createColumnsAndTypes() { Pair.with(EXEC_ID, VARCHAR_200), Pair.with(JOB_ID, VARCHAR_200), Pair.with(JOB_GROUP, VARCHAR_200))); + + // AUTOMATION_RUNS, AUTOMATION_NODE_OUTPUTS, AUTOMATION_FOREACH_ROWS - table/column DDL + // (CREATE TABLE, primary keys, indexes, addColumnIfNotExists migrations) is owned by + // AutomationDatabaseUtility.initialize() (called by SMSSWebWatcher), not this class - but + // every column must still be declared here too, or SelectQueryStruct-based reads against + // these tables fail with a NullPointerException resolving the conceptual->physical column + // name (AbstractSqlQueryUtil.isSelectorKeyword gets a null "selector" argument), since this + // OWL creator is this engine's only source of table/column metadata - there is no live + // schema-introspection fallback. Keep this column list in sync with AutomationDatabaseUtility's + // CREATE TABLE statements; needsRemake()/remakeOwl() automatically pick up any column added + // here on the next server startup, no manual OWL file deletion required. + addTable(AutomationConstants.TABLE_AUTOMATION_RUNS, Arrays.asList( + Pair.with(AutomationConstants.RUN_ID, VARCHAR_255), + Pair.with(AutomationConstants.PROJECT_ID, VARCHAR_255), + Pair.with(AutomationConstants.AUTOMATION_ID, VARCHAR_255), + Pair.with(AutomationConstants.STATUS, VARCHAR_200), + Pair.with(AutomationConstants.TRIGGER_TYPE, VARCHAR_200), + Pair.with(AutomationConstants.RESUMED_FROM_RUN, VARCHAR_255), + Pair.with(AutomationConstants.STARTED_AT, TIMESTAMP), + Pair.with(AutomationConstants.COMPLETED_AT, TIMESTAMP), + Pair.with(AutomationConstants.FAILED_NODE_ID, VARCHAR_255), + Pair.with(AutomationConstants.ERROR_MESSAGE, CLOB), + Pair.with(AutomationConstants.LAST_HEARTBEAT, TIMESTAMP), + Pair.with(AutomationConstants.TOTAL_NODES, INTEGER), + Pair.with(AutomationConstants.COMPLETED_NODES, INTEGER), + Pair.with(AutomationConstants.CREATED_BY, VARCHAR_255), + Pair.with(AutomationConstants.PARENT_RUN_ID, VARCHAR_255), + Pair.with(AutomationConstants.PARENT_NODE_ID, VARCHAR_255), + Pair.with(AutomationConstants.CANCEL_REQUESTED, BOOLEAN))); + + addTable(AutomationConstants.TABLE_AUTOMATION_NODE_OUTPUTS, Arrays.asList( + Pair.with(AutomationConstants.RUN_ID, VARCHAR_255), + Pair.with(AutomationConstants.NODE_ID, VARCHAR_255), + Pair.with(AutomationConstants.NODE_LABEL, VARCHAR_512), + Pair.with(AutomationConstants.EXECUTION_ORDER, INTEGER), + Pair.with(AutomationConstants.STATUS, VARCHAR_200), + Pair.with(AutomationConstants.STARTED_AT, TIMESTAMP), + Pair.with(AutomationConstants.COMPLETED_AT, TIMESTAMP), + Pair.with(AutomationConstants.DURATION_MS, BIGINT), + Pair.with(AutomationConstants.OUTPUT_VAR, VARCHAR_255), + Pair.with(AutomationConstants.OUTPUT_VALUE, CLOB), + Pair.with(AutomationConstants.OUTPUT_PREVIEW, VARCHAR_2000), + Pair.with(AutomationConstants.ROW_COUNT, INTEGER), + Pair.with(AutomationConstants.ERROR_MESSAGE, CLOB))); + + addTable(AutomationConstants.TABLE_AUTOMATION_FOREACH_ROWS, Arrays.asList( + Pair.with(AutomationConstants.RUN_ID, VARCHAR_255), + Pair.with(AutomationConstants.NODE_ID, VARCHAR_255), + Pair.with(AutomationConstants.ROW_INDEX, INTEGER), + Pair.with(AutomationConstants.ROW_KEY, VARCHAR_1000), + Pair.with(AutomationConstants.STATUS, VARCHAR_200), + Pair.with(AutomationConstants.STARTED_AT, TIMESTAMP), + Pair.with(AutomationConstants.COMPLETED_AT, TIMESTAMP), + Pair.with(AutomationConstants.DURATION_MS, BIGINT), + Pair.with(AutomationConstants.ERROR_MESSAGE, CLOB))); // @formatter:on } diff --git a/src/prerna/util/SMSSWebWatcher.java b/src/prerna/util/SMSSWebWatcher.java index fb8d3df84bf..a53961374e0 100644 --- a/src/prerna/util/SMSSWebWatcher.java +++ b/src/prerna/util/SMSSWebWatcher.java @@ -48,6 +48,7 @@ import prerna.masterdatabase.utility.MasterDatabaseUtility; import prerna.notifications.NotificationDbUtils; import prerna.prompt.PromptUtils; +import prerna.reactor.automation.AutomationDatabaseUtility; import prerna.reactor.scheduler.SchedulerDatabaseUtility; import prerna.theme.AbstractThemeUtils; import prerna.usertracking.UserTrackingUtils; @@ -216,6 +217,13 @@ public void init() { } catch (Exception e) { classLogger.error("Failed to load and start the scheduler database", e); } + + try { + AutomationDatabaseUtility.initialize(); + AutomationDatabaseUtility.markStaleRunsInterrupted(); + } catch (Exception e) { + classLogger.error("Failed to initialize automation engine tables", e); + } } } diff --git a/src/prerna/util/SystemEngineRegistry.java b/src/prerna/util/SystemEngineRegistry.java index 46ca3f33c56..9ce0068afc3 100644 --- a/src/prerna/util/SystemEngineRegistry.java +++ b/src/prerna/util/SystemEngineRegistry.java @@ -86,8 +86,8 @@ public final class SystemEngineRegistry { private static final Set LOCAL_MASTER_DB_ALLOWED = Set.of("prerna.auth", "prerna.masterdatabase", "prerna.reactor.masterdatabase", "prerna.reactor.utils", "prerna.util", "prerna.web.conf"); - private static final Set SCHEDULER_DB_ALLOWED = Set.of("prerna.auth", "prerna.reactor.scheduler", - "prerna.util", "prerna.web.conf"); + private static final Set SCHEDULER_DB_ALLOWED = Set.of("prerna.auth", "prerna.reactor.automation", + "prerna.reactor.scheduler", "prerna.util", "prerna.web.conf"); private static final Set THEMING_DB_ALLOWED = Set.of("prerna.auth", "prerna.theme", "prerna.util", "prerna.web.conf"); From 3973239ba0d3a39a535e27f696db0c67d7b4b3b9 Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Thu, 23 Jul 2026 11:15:32 -0400 Subject: [PATCH 02/25] fix: restore reactors --- .../CheckAutomationPollTriggerReactor.java | 266 ++++++++++++++++++ ...enerateAutomationWebhookSecretReactor.java | 152 ++++++++++ 2 files changed, 418 insertions(+) create mode 100644 src/prerna/reactor/automation/CheckAutomationPollTriggerReactor.java create mode 100644 src/prerna/reactor/automation/GenerateAutomationWebhookSecretReactor.java diff --git a/src/prerna/reactor/automation/CheckAutomationPollTriggerReactor.java b/src/prerna/reactor/automation/CheckAutomationPollTriggerReactor.java new file mode 100644 index 00000000000..386bc77febf --- /dev/null +++ b/src/prerna/reactor/automation/CheckAutomationPollTriggerReactor.java @@ -0,0 +1,266 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.AssetUtility; + +/** + * Poll-trigger reactor — called on a Quartz cron to check whether storage or + * database state has changed since the last run. Fires {@code TriggerAutomation} + * if a change is detected, then persists the new "last-seen" hash. + * + *

This reactor is designed to be used as the recipe for a Quartz scheduled + * job (registered via {@code ScheduleJob(...)}) with the recipe: + * 

+ *   CheckAutomationPollTrigger(project=["projectId"], type=["storage-poll"]);
+ *
+ * or + * 
+ *   CheckAutomationPollTrigger(project=["projectId"], type=["db-poll"]);
+ *
+ * + *

The poll configuration (engineId, path/query) is read from the automation's + * trigger node config inside {@code automation.json}. The "last seen" state hash + * is persisted in {@code automation-poll-state.json} in the portals folder. + * + *

State file format: + * 

+ *   { "storage-poll": "<sha256 of ListStoragePath output>",
+ *     "db-poll":      "<sha256 of SQL result>" }
+ *
+ */ +public class CheckAutomationPollTriggerReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(CheckAutomationPollTriggerReactor.class); + private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); + + private static final String POLL_STATE_FILE = "automation-poll-state.json"; + + public CheckAutomationPollTriggerReactor() { + this.keysToGet = new String[]{ "project", "type" }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + String pollType = this.keyValue.get(this.keysToGet[1]); // "storage-poll" or "db-poll" + + if (projectId == null || projectId.isBlank()) { + throw new IllegalArgumentException("Must provide a project id"); + } + if (pollType == null || pollType.isBlank()) { + throw new IllegalArgumentException("Must provide a poll type (storage-poll or db-poll)"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + + // Load trigger config from automation.json + Map triggerConfig = loadTriggerConfig(projectId); + if (triggerConfig == null) { + classLogger.warn("No trigger config found for project {}", projectId); + return noChange("No trigger config found"); + } + + // Execute the check pixel based on poll type + String currentResult; + String triggerType; + if ("storage-poll".equals(pollType)) { + currentResult = executeStorageCheck(triggerConfig); + triggerType = AutomationConstants.TRIGGER_STORAGE_POLL; + } else if ("db-poll".equals(pollType)) { + currentResult = executeDbCheck(triggerConfig); + triggerType = AutomationConstants.TRIGGER_DB_POLL; + } else { + throw new IllegalArgumentException("Unknown poll type: " + pollType + ". Expected storage-poll or db-poll"); + } + + if (currentResult == null) { + return noChange("Check pixel returned no result"); + } + + String currentHash = sha256(currentResult); + Map state = loadPollState(projectId); + String previousHash = state.getOrDefault(pollType, ""); + + if (currentHash.equals(previousHash)) { + classLogger.debug("Poll trigger for project {} ({}): no change detected", projectId, pollType); + return noChange("No change detected"); + } + + // State changed — fire automation + classLogger.info("Poll trigger for project {} ({}): change detected, firing automation", projectId, pollType); + state.put(pollType, currentHash); + savePollState(projectId, state); + + // Fire TriggerAutomation with the appropriate trigger type + String pixel = "TriggerAutomation(project=[\"" + projectId + "\"], triggerType=[\"" + triggerType + "\"]);"; + try { + this.insight.runPixel(pixel); + } catch (Exception e) { + classLogger.error("Failed to fire automation for project {} after change detected: {}", projectId, e.getMessage(), e); + throw new IllegalStateException("Change detected but automation trigger failed: " + e.getMessage(), e); + } + + Map result = new LinkedHashMap<>(); + result.put("triggered", true); + result.put("pollType", pollType); + result.put("projectId", projectId); + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + @SuppressWarnings("unchecked") + private Map loadTriggerConfig(String projectId) { + try { + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File automationFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); + if (!automationFile.exists()) return null; + + String json = Files.readString(automationFile.toPath(), StandardCharsets.UTF_8); + Map doc = GSON.fromJson(json, new TypeToken>(){}.getType()); + Map graph = (Map) doc.get("graph"); + if (graph == null) return null; + List> nodes = (List>) graph.get("nodes"); + if (nodes == null) return null; + + for (Map node : nodes) { + if (AutomationConstants.NODE_TRIGGER.equals(node.get("type"))) { + Object cfg = node.get("config"); + if (cfg instanceof Map) return (Map) cfg; + } + } + } catch (Exception e) { + classLogger.warn("Failed to load trigger config for {}: {}", projectId, e.getMessage()); + } + return null; + } + + private String executeStorageCheck(Map triggerConfig) { + String engineId = str(triggerConfig.get("storagePollEngineId")); + String path = str(triggerConfig.get("storagePollPath")); + if (engineId == null || path == null) { + classLogger.warn("Storage poll config incomplete: engineId={}, path={}", engineId, path); + return null; + } + String pixel = "ListStoragePath(storage=[\"" + engineId + "\"], storagePath=[\"" + path + "\"]);"; + try { + Object out = this.insight.runPixel(pixel); + return out != null ? GSON.toJson(out) : null; + } catch (Exception e) { + classLogger.warn("Storage poll check failed: {}", e.getMessage()); + return null; + } + } + + private String executeDbCheck(Map triggerConfig) { + String engineId = str(triggerConfig.get("dbPollEngineId")); + String query = str(triggerConfig.get("dbPollQuery")); + if (engineId == null || query == null) { + classLogger.warn("DB poll config incomplete: engineId={}, query={}", engineId, query); + return null; + } + String pixel = "SqlQuery(database=[\"" + engineId + "\"], query=[\"" + query + "\"]);"; + try { + Object out = this.insight.runPixel(pixel); + return out != null ? GSON.toJson(out) : null; + } catch (Exception e) { + classLogger.warn("DB poll check failed: {}", e.getMessage()); + return null; + } + } + + @SuppressWarnings("unchecked") + private Map loadPollState(String projectId) { + try { + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File stateFile = new File(portalsFolder + "/" + POLL_STATE_FILE); + if (!stateFile.exists()) return new HashMap<>(); + String json = Files.readString(stateFile.toPath(), StandardCharsets.UTF_8); + Map state = GSON.fromJson(json, new TypeToken>(){}.getType()); + return state != null ? state : new HashMap<>(); + } catch (Exception e) { + classLogger.warn("Could not load poll state for {}: {}", projectId, e.getMessage()); + return new HashMap<>(); + } + } + + private void savePollState(String projectId, Map state) { + try { + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File stateFile = new File(portalsFolder + "/" + POLL_STATE_FILE); + stateFile.getParentFile().mkdirs(); + Files.writeString(stateFile.toPath(), GSON.toJson(state), StandardCharsets.UTF_8); + } catch (Exception e) { + classLogger.warn("Could not save poll state for {}: {}", projectId, e.getMessage()); + } + } + + private static String sha256(String input) { + try { + MessageDigest md = MessageDigest.getInstance("SHA-256"); + byte[] bytes = md.digest(input.getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(); + for (byte b : bytes) sb.append(String.format("%02x", b)); + return sb.toString(); + } catch (Exception e) { + return String.valueOf(input.hashCode()); + } + } + + private static String str(Object v) { + return (v != null && !v.toString().isBlank()) ? v.toString() : null; + } + + private static NounMetadata noChange(String reason) { + Map r = new LinkedHashMap<>(); + r.put("triggered", false); + r.put("reason", reason); + return new NounMetadata(r, PixelDataType.MAP, PixelOperationType.OPERATION); + } +} diff --git a/src/prerna/reactor/automation/GenerateAutomationWebhookSecretReactor.java b/src/prerna/reactor/automation/GenerateAutomationWebhookSecretReactor.java new file mode 100644 index 00000000000..def6820a57e --- /dev/null +++ b/src/prerna/reactor/automation/GenerateAutomationWebhookSecretReactor.java @@ -0,0 +1,152 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; + +import prerna.auth.AccessToken; +import prerna.auth.AuthProvider; +import prerna.auth.User; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.AssetUtility; + +/** + * Generates (or regenerates) a webhook secret for an automation project. + * + *

Pixel: {@code GenerateAutomationWebhookSecret(project=["projectId"])} + * + *

Stores the secret in {@code automation-config.json} under the key + * {@code WEBHOOK_SECRET} (marked sensitive=true). Returns the plain-text + * secret once — it is not retrievable again from the API. + */ +public class GenerateAutomationWebhookSecretReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GenerateAutomationWebhookSecretReactor.class); + private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); + + private static final String WEBHOOK_SECRET_KEY = "WEBHOOK_SECRET"; + private static final String WEBHOOK_USER_KEY = "WEBHOOK_USER"; + + public GenerateAutomationWebhookSecretReactor() { + this.keysToGet = new String[]{ "project" }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + if (projectId == null || projectId.isBlank()) { + throw new IllegalArgumentException("Must provide a project id"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have edit access"); + } + + String secret = UUID.randomUUID().toString().replace("-", "") + + UUID.randomUUID().toString().replace("-", ""); + + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File configFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_CONFIG_FILE_NAME); + + List> entries = new ArrayList<>(); + if (configFile.exists()) { + try { + String json = Files.readString(configFile.toPath(), StandardCharsets.UTF_8); + List> existing = GSON.fromJson(json, + new TypeToken>>(){}.getType()); + if (existing != null) { + // copy all entries except the existing WEBHOOK_SECRET / WEBHOOK_USER + for (Map e : existing) { + Object key = e.get("key"); + if (!WEBHOOK_SECRET_KEY.equals(key) && !WEBHOOK_USER_KEY.equals(key)) { + entries.add(e); + } + } + } + } catch (Exception e) { + classLogger.warn("Could not parse existing automation config for {}: {}", projectId, e.getMessage()); + } + } + + // Build "PROVIDER:id,PROVIDER:id" string for the executing user + User callingUser = this.insight.getUser(); + StringBuilder userAccessBuilder = new StringBuilder(); + for (AuthProvider provider : callingUser.getLogins()) { + AccessToken token = callingUser.getAccessToken(provider); + if (token != null) { + if (userAccessBuilder.length() > 0) userAccessBuilder.append(","); + userAccessBuilder.append(provider.name()).append(":").append(token.getId()); + } + } + + Map secretEntry = new LinkedHashMap<>(); + secretEntry.put("key", WEBHOOK_SECRET_KEY); + secretEntry.put("value", secret); + secretEntry.put("sensitive", true); + entries.add(secretEntry); + + Map userEntry = new LinkedHashMap<>(); + userEntry.put("key", WEBHOOK_USER_KEY); + userEntry.put("value", userAccessBuilder.toString()); + userEntry.put("sensitive", true); + entries.add(userEntry); + + try { + configFile.getParentFile().mkdirs(); + Files.writeString(configFile.toPath(), GSON.toJson(entries), StandardCharsets.UTF_8); + } catch (Exception e) { + throw new IllegalStateException("Failed to save webhook secret: " + e.getMessage(), e); + } + + Map result = new LinkedHashMap<>(); + result.put("secret", secret); + result.put("projectId", projectId); + result.put("note", "Store this secret securely - it cannot be retrieved again. Pass it in the X-Webhook-Secret header when calling the webhook endpoint."); + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } +} From 98d0bc70bfc28ed3d42871d66577d671f6af1d6b Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Thu, 23 Jul 2026 17:46:06 -0400 Subject: [PATCH 03/25] fix: descope --- .../AutomationConditionEvaluator.java | 446 ------------------ .../automation/AutomationConstants.java | 57 +-- .../automation/AutomationDatabaseUtility.java | 267 +---------- .../automation/AutomationExecutionUtils.java | 12 - .../CheckAutomationPollTriggerReactor.java | 266 ----------- ...enerateAutomationWebhookSecretReactor.java | 152 ------ .../automation/GetAutomationRunReactor.java | 34 -- .../automation/PixelExecutionUtils.java | 232 --------- .../ResumeAutomationRunReactor.java | 113 ----- .../automation/RunAutomationNodeReactor.java | 64 ++- .../automation/TriggerAutomationReactor.java | 319 ++----------- .../nodes/AutomationNodeContext.java | 64 +-- .../nodes/AutomationNodeExecutors.java | 50 ++ .../nodes/ChildAutomationRunner.java | 70 --- .../nodes/DatabaseEngineNodeExecutor.java | 99 ++-- .../automation/nodes/EngineNodeSupport.java | 172 ------- .../nodes/FunctionEngineNodeExecutor.java | 68 +-- .../nodes/ModelEngineNodeExecutor.java | 123 +++-- .../automation/nodes/NodeDispatcher.java | 60 --- .../automation/nodes/PixelNodeExecutor.java | 101 ---- .../nodes/StorageEngineNodeExecutor.java | 103 ++-- .../nodes/VectorEngineNodeExecutor.java | 120 +++-- .../automation/nodes/WaitNodeExecutor.java | 2 +- .../scheduler/SchedulerOwlCreator.java | 17 +- 24 files changed, 413 insertions(+), 2598 deletions(-) delete mode 100644 src/prerna/reactor/automation/AutomationConditionEvaluator.java delete mode 100644 src/prerna/reactor/automation/CheckAutomationPollTriggerReactor.java delete mode 100644 src/prerna/reactor/automation/GenerateAutomationWebhookSecretReactor.java delete mode 100644 src/prerna/reactor/automation/PixelExecutionUtils.java delete mode 100644 src/prerna/reactor/automation/ResumeAutomationRunReactor.java create mode 100644 src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java delete mode 100644 src/prerna/reactor/automation/nodes/ChildAutomationRunner.java delete mode 100644 src/prerna/reactor/automation/nodes/EngineNodeSupport.java delete mode 100644 src/prerna/reactor/automation/nodes/NodeDispatcher.java delete mode 100644 src/prerna/reactor/automation/nodes/PixelNodeExecutor.java diff --git a/src/prerna/reactor/automation/AutomationConditionEvaluator.java b/src/prerna/reactor/automation/AutomationConditionEvaluator.java deleted file mode 100644 index e44ad137de4..00000000000 --- a/src/prerna/reactor/automation/AutomationConditionEvaluator.java +++ /dev/null @@ -1,446 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.reactor.automation; - -/** - * Safe, dependency-free evaluator for automation conditional / while-loop expressions - * and set-variable arithmetic. - * - *

Replaces the previous {@code javax.script} (JavaScript) evaluation. That path - * ran attacker-influenceable data (prior node outputs, HTTP/LLM responses substituted - * into the expression by {@link AutomationExecutionUtils#resolve}) through a scripting - * engine, which is a remote-code-execution vector once any JS engine is on the - * classpath; and on a JDK with no JS engine it silently degraded to an always-true - * check, so conditions never actually evaluated. This evaluator does neither: it - * parses a small, fixed grammar and can only ever return a value - it cannot reach - * Java classes, the filesystem, or the network. - * - *

Supported grammar (after {@code ${var}} substitution): - *

    - *
  • Logical: {@code || && !}
    • - *
  • Equality: {@code == === != !==}
    • - *
  • Relational: {@code < <= > >=}
    • - *
  • Arithmetic: {@code + - * / %} and unary {@code -}, with parentheses
    • - *
  • Literals: numbers, {@code "..."} / {@code '...'} strings, {@code true false null}
    • - *
  • A bare word (no operators) is treated as a string operand
    • - *
- * - *

Numeric comparisons are used when both operands parse as numbers; otherwise - * string comparison is used. Truthiness follows the automation convention: a string is - * falsy when empty or equal (ignoring case) to {@code "false"} / {@code "null"} / - * {@code "0"}, truthy otherwise. - */ -public final class AutomationConditionEvaluator { - - private AutomationConditionEvaluator() { - // utility class - } - - /** - * Evaluates {@code expression} and returns its truthiness. If the expression is not - * a parseable expression (e.g. plain free text), falls back to treating the whole - * trimmed string by the truthiness convention rather than failing the node. - * - * @param expression the fully resolved expression (no {@code ${var}} tokens remaining) - * @return the boolean result - */ - public static boolean toBoolean(String expression) { - if (expression == null) { - return false; - } - try { - Object value = new Parser(expression).parse(); - return truthy(value); - } catch (ParseException e) { - // Not an expression we understand - preserve the legacy "truthy string" behavior. - return truthyString(expression.trim()); - } - } - - /** - * Evaluates {@code expression} as arithmetic and returns the numeric result, or - * {@code null} if it is not a pure numeric expression. - * - * @param expression the fully resolved expression - * @return the numeric result, or {@code null} if non-numeric / unparseable - */ - public static Double toNumber(String expression) { - if (expression == null) { - return null; - } - try { - Object value = new Parser(expression).parse(); - return asNumber(value); - } catch (ParseException e) { - return null; - } - } - - // -- Truthiness / coercion ------------------------------------------------------- - - private static boolean truthy(Object v) { - if (v == null) { - return false; - } - if (v instanceof Boolean) { - return (Boolean) v; - } - if (v instanceof Double) { - double d = (Double) v; - return d != 0.0 && !Double.isNaN(d); - } - return truthyString(v.toString().trim()); - } - - private static boolean truthyString(String s) { - return !s.isEmpty() && !"false".equalsIgnoreCase(s) - && !"null".equalsIgnoreCase(s) && !"0".equals(s); - } - - private static Double asNumber(Object v) { - if (v instanceof Double) { - return (Double) v; - } - if (v instanceof Boolean) { - return ((Boolean) v) ? 1.0 : 0.0; - } - if (v instanceof String) { - String s = ((String) v).trim(); - if (s.isEmpty()) { - return null; - } - try { - return Double.parseDouble(s); - } catch (NumberFormatException e) { - return null; - } - } - return null; - } - - private static String asString(Object v) { - return v == null ? "null" : v.toString(); - } - - // -- Recursive-descent parser ---------------------------------------------------- - - /** Thrown internally when the input is not a valid expression. */ - private static final class ParseException extends RuntimeException { - private static final long serialVersionUID = 1L; - - ParseException(String message) { - super(message); - } - } - - private static final class Parser { - - private final String src; - private int pos; - - Parser(String src) { - this.src = src; - } - - Object parse() { - Object result = parseOr(); - skipWhitespace(); - if (this.pos < this.src.length()) { - throw new ParseException("Unexpected trailing input at position " + this.pos); - } - return result; - } - - private Object parseOr() { - Object left = parseAnd(); - while (match("||")) { - boolean l = truthy(left); - Object right = parseAnd(); - left = l || truthy(right); - } - return left; - } - - private Object parseAnd() { - Object left = parseEquality(); - while (match("&&")) { - boolean l = truthy(left); - Object right = parseEquality(); - left = l && truthy(right); - } - return left; - } - - private Object parseEquality() { - Object left = parseRelational(); - while (true) { - if (match("===")) { - left = strictEquals(left, parseRelational()); - } else if (match("!==")) { - left = !strictEquals(left, parseRelational()); - } else if (match("==")) { - left = looseEquals(left, parseRelational()); - } else if (match("!=")) { - left = !looseEquals(left, parseRelational()); - } else { - break; - } - } - return left; - } - - private Object parseRelational() { - Object left = parseAdditive(); - while (true) { - String op = matchAny("<=", ">=", "<", ">"); - if (op == null) { - break; - } - Object right = parseAdditive(); - left = compare(left, right, op); - } - return left; - } - - private Object parseAdditive() { - Object left = parseMultiplicative(); - while (true) { - String op = matchAny("+", "-"); - if (op == null) { - break; - } - Object right = parseMultiplicative(); - Double ln = asNumber(left); - Double rn = asNumber(right); - if ("+".equals(op)) { - // numeric add when both numeric, else string concatenation - left = (ln != null && rn != null) ? (Object) (ln + rn) - : (Object) (asString(left) + asString(right)); - } else { - left = requireNumber(ln, op) - requireNumber(rn, op); - } - } - return left; - } - - private Object parseMultiplicative() { - Object left = parseUnary(); - while (true) { - String op = matchAny("*", "/", "%"); - if (op == null) { - break; - } - double l = requireNumber(asNumber(left), op); - double r = requireNumber(asNumber(parseUnary()), op); - switch (op) { - case "*": left = l * r; break; - case "/": left = l / r; break; - default: left = l % r; break; - } - } - return left; - } - - private Object parseUnary() { - if (match("!")) { - return !truthy(parseUnary()); - } - if (match("-")) { - return -requireNumber(asNumber(parseUnary()), "-"); - } - return parsePrimary(); - } - - private Object parsePrimary() { - skipWhitespace(); - if (this.pos >= this.src.length()) { - throw new ParseException("Unexpected end of expression"); - } - char c = this.src.charAt(this.pos); - if (c == '(') { - this.pos++; - Object inner = parseOr(); - skipWhitespace(); - if (!match(")")) { - throw new ParseException("Expected ')'"); - } - return inner; - } - if (c == '"' || c == '\'') { - return readString(c); - } - if (Character.isDigit(c) || (c == '.' && peekDigit(1))) { - return readNumber(); - } - if (Character.isLetter(c) || c == '_' || c == '$') { - return readWord(); - } - throw new ParseException("Unexpected character '" + c + "' at position " + this.pos); - } - - private Object readString(char quote) { - this.pos++; // opening quote - StringBuilder sb = new StringBuilder(); - while (this.pos < this.src.length()) { - char c = this.src.charAt(this.pos++); - if (c == '\\' && this.pos < this.src.length()) { - char n = this.src.charAt(this.pos++); - switch (n) { - case 'n': sb.append('\n'); break; - case 't': sb.append('\t'); break; - case 'r': sb.append('\r'); break; - default: sb.append(n); break; - } - } else if (c == quote) { - return sb.toString(); - } else { - sb.append(c); - } - } - throw new ParseException("Unterminated string literal"); - } - - private Object readNumber() { - int start = this.pos; - while (this.pos < this.src.length()) { - char c = this.src.charAt(this.pos); - if (Character.isDigit(c) || c == '.' || c == 'e' || c == 'E' - || ((c == '+' || c == '-') && this.pos > start - && (this.src.charAt(this.pos - 1) == 'e' || this.src.charAt(this.pos - 1) == 'E'))) { - this.pos++; - } else { - break; - } - } - try { - return Double.parseDouble(this.src.substring(start, this.pos)); - } catch (NumberFormatException e) { - throw new ParseException("Invalid number literal"); - } - } - - private Object readWord() { - int start = this.pos; - while (this.pos < this.src.length()) { - char c = this.src.charAt(this.pos); - if (Character.isLetterOrDigit(c) || c == '_' || c == '$') { - this.pos++; - } else { - break; - } - } - String word = this.src.substring(start, this.pos); - if ("true".equals(word)) { - return Boolean.TRUE; - } - if ("false".equals(word)) { - return Boolean.FALSE; - } - if ("null".equals(word)) { - return null; - } - return word; // bare word treated as a string operand - } - - // -- token helpers ----------------------------------------------------------- - - private boolean match(String token) { - skipWhitespace(); - if (this.src.startsWith(token, this.pos)) { - this.pos += token.length(); - return true; - } - return false; - } - - private String matchAny(String... tokens) { - for (String t : tokens) { - if (match(t)) { - return t; - } - } - return null; - } - - private void skipWhitespace() { - while (this.pos < this.src.length() && Character.isWhitespace(this.src.charAt(this.pos))) { - this.pos++; - } - } - - private boolean peekDigit(int ahead) { - int i = this.pos + ahead; - return i < this.src.length() && Character.isDigit(this.src.charAt(i)); - } - } - - // -- comparison helpers ---------------------------------------------------------- - - private static double requireNumber(Double d, String op) { - if (d == null) { - throw new ParseException("Operator '" + op + "' requires a numeric operand"); - } - return d; - } - - private static boolean compare(Object left, Object right, String op) { - Double ln = asNumber(left); - Double rn = asNumber(right); - int cmp; - if (ln != null && rn != null) { - cmp = Double.compare(ln, rn); - } else { - cmp = asString(left).compareTo(asString(right)); - } - switch (op) { - case "<": return cmp < 0; - case "<=": return cmp <= 0; - case ">": return cmp > 0; - default: return cmp >= 0; // ">=" - } - } - - private static boolean looseEquals(Object a, Object b) { - Double an = asNumber(a); - Double bn = asNumber(b); - if (an != null && bn != null) { - return an.doubleValue() == bn.doubleValue(); - } - return asString(a).equals(asString(b)); - } - - private static boolean strictEquals(Object a, Object b) { - if (a == null || b == null) { - return a == b; - } - if (a.getClass() != b.getClass()) { - return false; - } - return a.equals(b); - } -} diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index 7fa8d450129..3416f1eb557 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -27,10 +27,6 @@ *******************************************************************************/ package prerna.reactor.automation; -/** - * Shared constants for the Automation Engine subsystem. - * Covers table/column names, status values, and node types. - */ public class AutomationConstants { private AutomationConstants() {} @@ -40,20 +36,12 @@ private AutomationConstants() {} public static final String AUTOMATION_FILE_NAME = "automation.json"; public static final String AUTOMATION_CONFIG_FILE_NAME = "automation-config.json"; - /** Placeholder returned by GetAutomationConfig in place of a sensitive value; never persisted back. */ public static final String SENSITIVE_MASK = "***"; // -- DB Table names ------------------------------------------------------------ public static final String TABLE_AUTOMATION_RUNS = "AUTOMATION_RUNS"; public static final String TABLE_AUTOMATION_NODE_OUTPUTS = "AUTOMATION_NODE_OUTPUTS"; - public static final String TABLE_AUTOMATION_FOREACH_ROWS = "AUTOMATION_FOREACH_ROWS"; - /** - * Single-row-per-project marker table enforcing "at most one active run per project" - * cluster-wide, via a primary key on PROJECT_ID. Claiming a row is an atomic INSERT - * (fails with a constraint violation if another run already holds it); the row is - * released on any terminal run status. - */ public static final String TABLE_AUTOMATION_ACTIVE_RUN = "AUTOMATION_ACTIVE_RUN"; // -- AUTOMATION_RUNS columns --------------------------------------------------- @@ -63,7 +51,6 @@ private AutomationConstants() {} public static final String AUTOMATION_ID = "AUTOMATION_ID"; public static final String STATUS = "STATUS"; public static final String TRIGGER_TYPE = "TRIGGER_TYPE"; - public static final String RESUMED_FROM_RUN = "RESUMED_FROM_RUN"; public static final String STARTED_AT = "STARTED_AT"; public static final String COMPLETED_AT = "COMPLETED_AT"; public static final String FAILED_NODE_ID = "FAILED_NODE_ID"; @@ -72,13 +59,6 @@ private AutomationConstants() {} public static final String TOTAL_NODES = "TOTAL_NODES"; public static final String COMPLETED_NODES = "COMPLETED_NODES"; public static final String CREATED_BY = "CREATED_BY"; - public static final String PARENT_RUN_ID = "PARENT_RUN_ID"; - public static final String PARENT_NODE_ID = "PARENT_NODE_ID"; - /** - * Cluster-safe cancellation flag. Set by CancelAutomationRunReactor regardless of which - * pod receives the cancel request; polled by the executing pod's between-node check - * alongside the in-memory (same-pod fast path) AtomicBoolean. - */ public static final String CANCEL_REQUESTED = "CANCEL_REQUESTED"; // -- AUTOMATION_ACTIVE_RUN columns --------------------------------------------- @@ -94,12 +74,6 @@ private AutomationConstants() {} public static final String OUTPUT_VAR = "OUTPUT_VAR"; public static final String OUTPUT_VALUE = "OUTPUT_VALUE"; public static final String OUTPUT_PREVIEW = "OUTPUT_PREVIEW"; - public static final String ROW_COUNT = "ROW_COUNT"; - - // -- AUTOMATION_FOREACH_ROWS columns ------------------------------------------ - - public static final String ROW_INDEX = "ROW_INDEX"; - public static final String ROW_KEY = "ROW_KEY"; // -- Run statuses -------------------------------------------------------------- @@ -120,14 +94,8 @@ private AutomationConstants() {} // -- Trigger types ------------------------------------------------------------- public static final String TRIGGER_MANUAL = "MANUAL"; - public static final String TRIGGER_SCHEDULED = "SCHEDULED"; - public static final String TRIGGER_RESUME = "RESUME"; - public static final String TRIGGER_SUB_AUTOMATION = "SUB_AUTOMATION"; - public static final String TRIGGER_WEBHOOK = "WEBHOOK"; - public static final String TRIGGER_STORAGE_POLL = "STORAGE_POLL"; - public static final String TRIGGER_DB_POLL = "DB_POLL"; - // -- Node types ---------------------------------------------------------------- + // -- Node types (Phase 1) ------------------------------------------------------ public static final String NODE_TRIGGER = "trigger"; public static final String NODE_DATABASE_ENGINE = "database-engine"; @@ -135,28 +103,7 @@ private AutomationConstants() {} public static final String NODE_VECTOR_ENGINE = "vector-engine"; public static final String NODE_MODEL_ENGINE = "model-engine"; public static final String NODE_FUNCTION_ENGINE = "function-engine"; - public static final String NODE_APP = "app"; - public static final String NODE_CUSTOM_PIXEL = "custom-pixel"; - public static final String NODE_FOR_EACH = "for-each"; - public static final String NODE_TRANSFORM = "transform"; - public static final String NODE_SUB_AUTOMATION = "sub-automation"; - public static final String NODE_CONDITIONAL = "conditional"; - public static final String NODE_WHILE_LOOP = "while-loop"; - public static final String NODE_TRY_CATCH = "try-catch"; public static final String NODE_WAIT = "wait"; - public static final String NODE_SET_VARIABLE = "set-variable"; - public static final String NODE_EMAIL = "email"; - public static final String NODE_HTTP_REQUEST = "http-request"; - public static final String NODE_NOTIFICATION = "notification"; - public static final String NODE_SWITCH = "switch"; - public static final String NODE_RETRY = "retry"; - public static final String NODE_PARALLEL = "parallel"; - - // -- Sub-automation node config keys ------------------------------------------ - - public static final String SUB_AUTOMATION_TARGET_PROJECT = "targetProjectId"; - public static final String SUB_AUTOMATION_INPUT_MAPPING = "inputMapping"; - public static final int MAX_SUB_AUTOMATION_DEPTH = 10; // -- Data type constants (for table creation) ---------------------------------- @@ -171,9 +118,7 @@ private AutomationConstants() {} // -- Defaults ------------------------------------------------------------------ public static final String DEFAULT_AUTOMATION_ID = "default"; - public static final int DEFAULT_TIMEOUT_SECONDS = 300; public static final int HEARTBEAT_INTERVAL_SECONDS = 30; public static final int STALE_HEARTBEAT_THRESHOLD_MINUTES = 5; - public static final int FOREACH_BATCH_SIZE = 100; public static final int OUTPUT_PREVIEW_MAX_LENGTH = 2000; } diff --git a/src/prerna/reactor/automation/AutomationDatabaseUtility.java b/src/prerna/reactor/automation/AutomationDatabaseUtility.java index e1443b9931f..3d6ce3c6440 100644 --- a/src/prerna/reactor/automation/AutomationDatabaseUtility.java +++ b/src/prerna/reactor/automation/AutomationDatabaseUtility.java @@ -29,7 +29,6 @@ import java.sql.Connection; import java.sql.PreparedStatement; -import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; import java.sql.Types; @@ -39,7 +38,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Date; -import java.util.HashMap; import java.util.List; import java.util.Map; @@ -51,8 +49,6 @@ import prerna.query.querystruct.filters.SimpleQueryFilter; import prerna.query.querystruct.selectors.QueryColumnOrderBySelector; import prerna.query.querystruct.selectors.QueryColumnSelector; -import prerna.query.querystruct.selectors.QueryFunctionHelper; -import prerna.query.querystruct.selectors.QueryFunctionSelector; import prerna.sablecc2.om.PixelDataType; import prerna.util.ConnectionUtils; import prerna.util.QueryExecutionUtility; @@ -62,7 +58,7 @@ /** * Database utility for the Automation Engine subsystem. - * Manages AUTOMATION_RUNS, AUTOMATION_NODE_OUTPUTS, and AUTOMATION_FOREACH_ROWS tables + * Manages AUTOMATION_RUNS, AUTOMATION_NODE_OUTPUTS, and AUTOMATION_ACTIVE_RUN tables * in the scheduler database. * * Follows the same patterns as {@link prerna.reactor.scheduler.SchedulerDatabaseUtility}. @@ -75,7 +71,6 @@ public final class AutomationDatabaseUtility { // Table name shortcuts for SelectQueryStruct (TABLE__COLUMN format) private static final String TABLE_RUNS = AutomationConstants.TABLE_AUTOMATION_RUNS; private static final String TABLE_NODE_OUTPUTS = AutomationConstants.TABLE_AUTOMATION_NODE_OUTPUTS; - private static final String TABLE_FOREACH = AutomationConstants.TABLE_AUTOMATION_FOREACH_ROWS; private AutomationDatabaseUtility() { // static utility - no instantiation @@ -86,10 +81,9 @@ private AutomationDatabaseUtility() { // AUTOMATION_RUNS private static final String INSERT_RUN = """ INSERT INTO AUTOMATION_RUNS \ - (RUN_ID, PROJECT_ID, AUTOMATION_ID, STATUS, TRIGGER_TYPE, RESUMED_FROM_RUN, \ - STARTED_AT, LAST_HEARTBEAT, TOTAL_NODES, COMPLETED_NODES, CREATED_BY, \ - PARENT_RUN_ID, PARENT_NODE_ID) \ - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?, ?)"""; + (RUN_ID, PROJECT_ID, AUTOMATION_ID, STATUS, TRIGGER_TYPE, \ + STARTED_AT, LAST_HEARTBEAT, TOTAL_NODES, COMPLETED_NODES, CREATED_BY) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?)"""; private static final String UPDATE_RUN_STATUS = """ UPDATE AUTOMATION_RUNS SET STATUS = ?, COMPLETED_AT = ?, \ @@ -123,7 +117,7 @@ private AutomationDatabaseUtility() { private static final String UPDATE_NODE_OUTPUT_SUCCESS = """ UPDATE AUTOMATION_NODE_OUTPUTS SET STATUS = ?, STARTED_AT = ?, COMPLETED_AT = ?, \ - DURATION_MS = ?, OUTPUT_VAR = ?, OUTPUT_VALUE = ?, OUTPUT_PREVIEW = ?, ROW_COUNT = ? \ + DURATION_MS = ?, OUTPUT_VAR = ?, OUTPUT_VALUE = ?, OUTPUT_PREVIEW = ? \ WHERE RUN_ID = ? AND NODE_ID = ?"""; private static final String UPDATE_NODE_OUTPUT_FAILED = """ @@ -133,19 +127,6 @@ private AutomationDatabaseUtility() { private static final String UPDATE_NODE_STATUS = "UPDATE AUTOMATION_NODE_OUTPUTS SET STATUS = ?, STARTED_AT = ? WHERE RUN_ID = ? AND NODE_ID = ?"; - // AUTOMATION_FOREACH_ROWS - private static final String INSERT_FOREACH_ROW = """ - INSERT INTO AUTOMATION_FOREACH_ROWS \ - (RUN_ID, NODE_ID, ROW_INDEX, ROW_KEY, STATUS, STARTED_AT, COMPLETED_AT, DURATION_MS, ERROR_MESSAGE) \ - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"""; - - // Aggregate queries (use PreparedStatement - CASE WHEN not easily expressed in SelectQueryStruct) - private static final String SELECT_FOREACH_PROGRESS = """ - SELECT COUNT(*) AS TOTAL, \ - SUM(CASE WHEN STATUS = 'SUCCESS' THEN 1 ELSE 0 END) AS SUCCEEDED, \ - SUM(CASE WHEN STATUS = 'FAILED' THEN 1 ELSE 0 END) AS FAILED \ - FROM AUTOMATION_FOREACH_ROWS WHERE RUN_ID = ? AND NODE_ID = ?"""; - // -- Initialization ------------------------------------------------------------ /** @@ -173,7 +154,6 @@ public static void initialize() { createAutomationRunsTable(conn, queryUtil, database, schema, allowIfExists, dateTimeType, clobType); createAutomationNodeOutputsTable(conn, queryUtil, database, schema, allowIfExists, dateTimeType, clobType); - createAutomationForEachRowsTable(conn, queryUtil, database, schema, allowIfExists, dateTimeType, clobType); createAutomationActiveRunTable(conn, queryUtil, database, schema, allowIfExists, dateTimeType); if (!conn.getAutoCommit()) { @@ -425,19 +405,7 @@ public static boolean isCancelRequested(String runId) { * Inserts a new automation run record. */ public static boolean insertRun(String runId, String projectId, String automationId, - String triggerType, String resumedFromRun, int totalNodes, String createdBy) { - return insertRun(runId, projectId, automationId, triggerType, resumedFromRun, - totalNodes, createdBy, null, null); - } - - /** - * Inserts a new automation run record, optionally linked to a parent run/node - used when - * a sub-automation node triggers another project's automation. {@code parentRunId} and - * {@code parentNodeId} are null for top-level (manual/scheduled/resume) runs. - */ - public static boolean insertRun(String runId, String projectId, String automationId, - String triggerType, String resumedFromRun, int totalNodes, String createdBy, - String parentRunId, String parentNodeId) { + String triggerType, int totalNodes, String createdBy) { IRDBMSEngine schedulerDb = getSchedulerDb(); if (schedulerDb == null) return false; @@ -453,13 +421,10 @@ public static boolean insertRun(String runId, String projectId, String automatio ps.setString(index++, automationId); ps.setString(index++, AutomationConstants.STATUS_RUNNING); ps.setString(index++, triggerType); - setNullableString(ps, index++, resumedFromRun); ps.setTimestamp(index++, now); ps.setTimestamp(index++, now); ps.setInt(index++, totalNodes); ps.setString(index++, createdBy); - setNullableString(ps, index++, parentRunId); - setNullableString(ps, index++, parentNodeId); ps.executeUpdate(); } @@ -581,7 +546,6 @@ public static List> getRunsForProject(String projectId, int qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__AUTOMATION_ID", "AUTOMATION_ID")); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__STATUS", "STATUS")); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__TRIGGER_TYPE", "TRIGGER_TYPE")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__RESUMED_FROM_RUN", "RESUMED_FROM_RUN")); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__STARTED_AT", "STARTED_AT")); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__COMPLETED_AT", "COMPLETED_AT")); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__FAILED_NODE_ID", "FAILED_NODE_ID")); @@ -612,7 +576,6 @@ public static Map getRunDetail(String runId) { qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__AUTOMATION_ID", "AUTOMATION_ID")); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__STATUS", "STATUS")); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__TRIGGER_TYPE", "TRIGGER_TYPE")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__RESUMED_FROM_RUN", "RESUMED_FROM_RUN")); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__STARTED_AT", "STARTED_AT")); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__COMPLETED_AT", "COMPLETED_AT")); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__FAILED_NODE_ID", "FAILED_NODE_ID")); @@ -736,7 +699,7 @@ public static boolean markNodeRunning(String runId, String nodeId) { * Updates a node output after successful execution. */ public static boolean updateNodeSuccess(String runId, String nodeId, Timestamp startedAt, - long durationMs, String outputVar, String outputValue, String outputPreview, Integer rowCount) { + long durationMs, String outputVar, String outputValue, String outputPreview) { IRDBMSEngine schedulerDb = getSchedulerDb(); if (schedulerDb == null) return false; @@ -755,11 +718,6 @@ public static boolean updateNodeSuccess(String runId, String nodeId, Timestamp s // Handle CLOB for potentially large output values queryUtil.handleInsertionOfClob(conn, ps, outputValue, index++, AutomationExecutionUtils.GSON); ps.setString(index++, outputPreview); - if (rowCount != null) { - ps.setInt(index++, rowCount); - } else { - ps.setNull(index++, Types.INTEGER); - } ps.setString(index++, runId); ps.setString(index++, nodeId); ps.executeUpdate(); @@ -832,7 +790,6 @@ public static List> getNodeOutputsForRun(String runId) { qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__OUTPUT_VAR", "OUTPUT_VAR")); qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__OUTPUT_VALUE", "OUTPUT_VALUE")); qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__OUTPUT_PREVIEW", "OUTPUT_PREVIEW")); - qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__ROW_COUNT", "ROW_COUNT")); qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__ERROR_MESSAGE", "ERROR_MESSAGE")); qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( @@ -844,138 +801,6 @@ public static List> getNodeOutputsForRun(String runId) { return results != null ? results : new ArrayList<>(); } - // -- AUTOMATION_FOREACH_ROWS CRUD ------------------------------------------------ - - /** - * Batch-inserts for-each row results. Called with batches of - * {@link AutomationConstants#FOREACH_BATCH_SIZE} rows during for-each execution. - */ - public static boolean insertForEachRowsBatch(String runId, String nodeId, - List rows) { - IRDBMSEngine schedulerDb = getSchedulerDb(); - if (schedulerDb == null) return false; - - Connection conn = null; - try { - conn = schedulerDb.getConnection(); - try (PreparedStatement ps = conn.prepareStatement(INSERT_FOREACH_ROW)) { - for (ForEachRowResult row : rows) { - int index = 1; - ps.setString(index++, runId); - ps.setString(index++, nodeId); - ps.setInt(index++, row.rowIndex()); - ps.setString(index++, row.rowKey()); - ps.setString(index++, row.status()); - ps.setTimestamp(index++, row.startedAt()); - ps.setTimestamp(index++, toTimestamp(Instant.now())); - ps.setLong(index++, row.durationMs()); - ps.setString(index++, row.errorMessage()); - ps.addBatch(); - } - ps.executeBatch(); - } - if (!conn.getAutoCommit()) { - conn.commit(); - } - return true; - } catch (SQLException e) { - classLogger.error("Failed to batch-insert for-each rows for run '{}', node '{}': {}", - runId, nodeId, e.getMessage(), e); - return false; - } finally { - closeConnection(schedulerDb, conn); - } - } - - /** - * Gets aggregate progress for a for-each node. - * Uses PreparedStatement directly because this query involves conditional - * aggregates (SUM with CASE WHEN) not easily expressed via SelectQueryStruct. - * - * @return map with keys "total", "succeeded", "failed" - */ - public static Map getForEachProgress(String runId, String nodeId) { - IRDBMSEngine schedulerDb = getSchedulerDb(); - Map progress = new HashMap<>(); - if (schedulerDb == null) return progress; - - Connection conn = null; - try { - conn = schedulerDb.getConnection(); - try (PreparedStatement ps = conn.prepareStatement(SELECT_FOREACH_PROGRESS)) { - ps.setString(1, runId); - ps.setString(2, nodeId); - try (ResultSet rs = ps.executeQuery()) { - if (rs.next()) { - progress.put("total", rs.getInt(1)); - progress.put("succeeded", rs.getInt(2)); - progress.put("failed", rs.getInt(3)); - } - } - } - } catch (SQLException e) { - classLogger.error("Failed to get for-each progress for run '{}', node '{}': {}", - runId, nodeId, e.getMessage(), e); - } finally { - closeConnection(schedulerDb, conn); - } - return progress; - } - - /** - * Gets the failed rows for a for-each node (for drill-down). - */ - public static List> getForEachFailures(String runId, String nodeId, int limit) { - IRDBMSEngine schedulerDb = getSchedulerDb(); - if (schedulerDb == null) return new ArrayList<>(); - - SelectQueryStruct qs = new SelectQueryStruct(); - qs.addSelector(new QueryColumnSelector(TABLE_FOREACH + "__ROW_INDEX", "ROW_INDEX")); - qs.addSelector(new QueryColumnSelector(TABLE_FOREACH + "__ROW_KEY", "ROW_KEY")); - qs.addSelector(new QueryColumnSelector(TABLE_FOREACH + "__ERROR_MESSAGE", "ERROR_MESSAGE")); - qs.addSelector(new QueryColumnSelector(TABLE_FOREACH + "__COMPLETED_AT", "COMPLETED_AT")); - - qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( - TABLE_FOREACH + "__RUN_ID", "==", runId, PixelDataType.CONST_STRING)); - qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( - TABLE_FOREACH + "__NODE_ID", "==", nodeId, PixelDataType.CONST_STRING)); - qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( - TABLE_FOREACH + "__STATUS", "==", AutomationConstants.NODE_STATUS_FAILED, PixelDataType.CONST_STRING)); - qs.addOrderBy(TABLE_FOREACH + "__ROW_INDEX", - QueryColumnOrderBySelector.ORDER_BY_DIRECTION.ASC.toString()); - qs.setLimit(limit); - - List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); - return results != null ? results : new ArrayList<>(); - } - - /** - * Gets the max row index already processed for a for-each node (for resume). - * - * @return the max row index, or -1 if no rows have been processed - */ - public static int getForEachLastProcessedIndex(String runId, String nodeId) { - IRDBMSEngine schedulerDb = getSchedulerDb(); - if (schedulerDb == null) return -1; - - SelectQueryStruct qs = new SelectQueryStruct(); - qs.addSelector(QueryFunctionSelector.makeFunctionSelector( - QueryFunctionHelper.MAX, TABLE_FOREACH + "__ROW_INDEX", "MAX_INDEX")); - qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( - TABLE_FOREACH + "__RUN_ID", "==", runId, PixelDataType.CONST_STRING)); - qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( - TABLE_FOREACH + "__NODE_ID", "==", nodeId, PixelDataType.CONST_STRING)); - - List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); - if (results != null && !results.isEmpty()) { - Object maxVal = results.get(0).get("MAX_INDEX"); - if (maxVal instanceof Number) { - return ((Number) maxVal).intValue(); - } - } - return -1; - } - // -- Table Creation ------------------------------------------------------------ private static void createAutomationRunsTable(Connection conn, AbstractSqlQueryUtil queryUtil, @@ -988,17 +813,17 @@ private static void createAutomationRunsTable(Connection conn, AbstractSqlQueryU } String[] colNames = { "RUN_ID", "PROJECT_ID", "AUTOMATION_ID", "STATUS", "TRIGGER_TYPE", - "RESUMED_FROM_RUN", "STARTED_AT", "COMPLETED_AT", "FAILED_NODE_ID", + "STARTED_AT", "COMPLETED_AT", "FAILED_NODE_ID", "ERROR_MESSAGE", "LAST_HEARTBEAT", "TOTAL_NODES", "COMPLETED_NODES", "CREATED_BY", - "PARENT_RUN_ID", "PARENT_NODE_ID", "CANCEL_REQUESTED" }; + "CANCEL_REQUESTED" }; String[] types = { "VARCHAR(255)", "VARCHAR(255)", "VARCHAR(255)", "VARCHAR(50)", "VARCHAR(50)", - "VARCHAR(255)", dateTimeType, dateTimeType, "VARCHAR(255)", + dateTimeType, dateTimeType, "VARCHAR(255)", clobType, dateTimeType, "INTEGER", "INTEGER", "VARCHAR(255)", - "VARCHAR(255)", "VARCHAR(255)", queryUtil.getBooleanDataTypeName() }; + queryUtil.getBooleanDataTypeName() }; String[] constraints = { "NOT NULL", "NOT NULL", null, "NOT NULL", "NOT NULL", - null, "NOT NULL", null, null, + "NOT NULL", null, null, null, null, null, null, null, - null, null, null }; + null }; String sql; if (allowIfExists) { @@ -1011,9 +836,7 @@ private static void createAutomationRunsTable(Connection conn, AbstractSqlQueryU ps.execute(); } - // Migrate installs whose AUTOMATION_RUNS predates sub-automation support / cluster-safe cancel - addColumnIfNotExists(conn, queryUtil, tableName, "PARENT_RUN_ID", "VARCHAR(255)"); - addColumnIfNotExists(conn, queryUtil, tableName, "PARENT_NODE_ID", "VARCHAR(255)"); + // Migrate installs that predate cluster-safe cancel addColumnIfNotExists(conn, queryUtil, tableName, "CANCEL_REQUESTED", queryUtil.getBooleanDataTypeName()); // Primary key @@ -1023,7 +846,6 @@ private static void createAutomationRunsTable(Connection conn, AbstractSqlQueryU createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_AR_PROJECT", tableName, new String[]{"PROJECT_ID"}); createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_AR_STATUS", tableName, new String[]{"PROJECT_ID", "STATUS"}); createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_AR_STARTED", tableName, new String[]{"PROJECT_ID", "STARTED_AT"}); - createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_AR_PARENT", tableName, new String[]{"PARENT_RUN_ID"}); } private static void createAutomationNodeOutputsTable(Connection conn, AbstractSqlQueryUtil queryUtil, @@ -1037,13 +859,13 @@ private static void createAutomationNodeOutputsTable(Connection conn, AbstractSq String[] colNames = { "RUN_ID", "NODE_ID", "NODE_LABEL", "EXECUTION_ORDER", "STATUS", "STARTED_AT", "COMPLETED_AT", "DURATION_MS", "OUTPUT_VAR", - "OUTPUT_VALUE", "OUTPUT_PREVIEW", "ROW_COUNT", "ERROR_MESSAGE" }; + "OUTPUT_VALUE", "OUTPUT_PREVIEW", "ERROR_MESSAGE" }; String[] types = { "VARCHAR(255)", "VARCHAR(255)", "VARCHAR(500)", "INTEGER", "VARCHAR(50)", dateTimeType, dateTimeType, "BIGINT", "VARCHAR(255)", - clobType, "VARCHAR(2000)", "INTEGER", clobType }; + clobType, "VARCHAR(2000)", clobType }; String[] constraints = { "NOT NULL", "NOT NULL", null, "NOT NULL", "NOT NULL", null, null, null, null, - null, null, null, null }; + null, null, null }; String sql; if (allowIfExists) { @@ -1063,41 +885,6 @@ private static void createAutomationNodeOutputsTable(Connection conn, AbstractSq createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_ANO_RUN", tableName, new String[]{"RUN_ID"}); } - private static void createAutomationForEachRowsTable(Connection conn, AbstractSqlQueryUtil queryUtil, - String database, String schema, boolean allowIfExists, String dateTimeType, String clobType) throws SQLException { - - String tableName = AutomationConstants.TABLE_AUTOMATION_FOREACH_ROWS; - - if (!allowIfExists && queryUtil.tableExists(conn, tableName, database, schema)) { - return; - } - - String[] colNames = { "RUN_ID", "NODE_ID", "ROW_INDEX", "ROW_KEY", "STATUS", - "STARTED_AT", "COMPLETED_AT", "DURATION_MS", "ERROR_MESSAGE" }; - String[] types = { "VARCHAR(255)", "VARCHAR(255)", "INTEGER", "VARCHAR(1000)", "VARCHAR(50)", - dateTimeType, dateTimeType, "BIGINT", clobType }; - String[] constraints = { "NOT NULL", "NOT NULL", "NOT NULL", null, "NOT NULL", - null, null, null, null }; - - String sql; - if (allowIfExists) { - sql = queryUtil.createTableIfNotExistsWithCustomConstraints(tableName, colNames, types, constraints); - } else { - sql = queryUtil.createTableWithCustomConstraints(tableName, colNames, types, constraints); - } - classLogger.info("Creating table {}: {}", tableName, sql); - try (PreparedStatement ps = conn.prepareStatement(sql)) { - ps.execute(); - } - - // Composite primary key - addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, "PK_AUTO_FE_ROWS", new String[]{"RUN_ID", "NODE_ID", "ROW_INDEX"}); - - // Indexes - createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_AFR_RUN_NODE", tableName, new String[]{"RUN_ID", "NODE_ID"}); - createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_AFR_STATUS", tableName, new String[]{"RUN_ID", "NODE_ID", "STATUS"}); - } - /** * Creates the AUTOMATION_ACTIVE_RUN marker table - a single row per project, keyed on * PROJECT_ID, used to atomically enforce "at most one active run per project" cluster-wide. @@ -1251,24 +1038,4 @@ private static void createIndexIfNotExists(Connection conn, AbstractSqlQueryUtil } } - // -- Data Transfer Object ------------------------------------------------------ - - /** - * Record for a single for-each row result, used in batch inserts. - */ - public record ForEachRowResult( - int rowIndex, - String rowKey, - String status, - String errorMessage, - Timestamp startedAt, - long durationMs - ) { - public ForEachRowResult(int rowIndex, String rowKey, String status, String errorMessage, long startTimeMs) { - this(rowIndex, rowKey, status, errorMessage, - Utility.getSqlTimestampUTC(LocalDateTime.ofInstant( - Instant.ofEpochMilli(startTimeMs), ZoneOffset.UTC)), - System.currentTimeMillis() - startTimeMs); - } - } } diff --git a/src/prerna/reactor/automation/AutomationExecutionUtils.java b/src/prerna/reactor/automation/AutomationExecutionUtils.java index 4a66f88d786..f2f02ed290a 100644 --- a/src/prerna/reactor/automation/AutomationExecutionUtils.java +++ b/src/prerna/reactor/automation/AutomationExecutionUtils.java @@ -90,18 +90,6 @@ public static String resolve(String template, Map scope, Map node) { - Object timeout = node.get("timeoutSeconds"); - if (timeout instanceof Number) { - return ((Number) timeout).intValue(); - } - return AutomationConstants.DEFAULT_TIMEOUT_SECONDS; - } - /** * Loads {@code automation-config.json} for a project and returns key->value pairs. * Returns an empty map if the file does not exist or cannot be parsed. diff --git a/src/prerna/reactor/automation/CheckAutomationPollTriggerReactor.java b/src/prerna/reactor/automation/CheckAutomationPollTriggerReactor.java deleted file mode 100644 index 386bc77febf..00000000000 --- a/src/prerna/reactor/automation/CheckAutomationPollTriggerReactor.java +++ /dev/null @@ -1,266 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.reactor.automation; - -import java.io.File; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.security.MessageDigest; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.reflect.TypeToken; - -import prerna.auth.utils.SecurityProjectUtils; -import prerna.reactor.AbstractReactor; -import prerna.sablecc2.om.PixelDataType; -import prerna.sablecc2.om.PixelOperationType; -import prerna.sablecc2.om.nounmeta.NounMetadata; -import prerna.util.AssetUtility; - -/** - * Poll-trigger reactor — called on a Quartz cron to check whether storage or - * database state has changed since the last run. Fires {@code TriggerAutomation} - * if a change is detected, then persists the new "last-seen" hash. - * - *

This reactor is designed to be used as the recipe for a Quartz scheduled - * job (registered via {@code ScheduleJob(...)}) with the recipe: - * 

- *   CheckAutomationPollTrigger(project=["projectId"], type=["storage-poll"]);
- *
- * or - * 
- *   CheckAutomationPollTrigger(project=["projectId"], type=["db-poll"]);
- *
- * - *

The poll configuration (engineId, path/query) is read from the automation's - * trigger node config inside {@code automation.json}. The "last seen" state hash - * is persisted in {@code automation-poll-state.json} in the portals folder. - * - *

State file format: - * 

- *   { "storage-poll": "<sha256 of ListStoragePath output>",
- *     "db-poll":      "<sha256 of SQL result>" }
- *
- */ -public class CheckAutomationPollTriggerReactor extends AbstractReactor { - - private static final Logger classLogger = LogManager.getLogger(CheckAutomationPollTriggerReactor.class); - private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); - - private static final String POLL_STATE_FILE = "automation-poll-state.json"; - - public CheckAutomationPollTriggerReactor() { - this.keysToGet = new String[]{ "project", "type" }; - } - - @Override - public NounMetadata execute() { - organizeKeys(); - String projectId = this.keyValue.get(this.keysToGet[0]); - String pollType = this.keyValue.get(this.keysToGet[1]); // "storage-poll" or "db-poll" - - if (projectId == null || projectId.isBlank()) { - throw new IllegalArgumentException("Must provide a project id"); - } - if (pollType == null || pollType.isBlank()) { - throw new IllegalArgumentException("Must provide a poll type (storage-poll or db-poll)"); - } - - projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); - - // Load trigger config from automation.json - Map triggerConfig = loadTriggerConfig(projectId); - if (triggerConfig == null) { - classLogger.warn("No trigger config found for project {}", projectId); - return noChange("No trigger config found"); - } - - // Execute the check pixel based on poll type - String currentResult; - String triggerType; - if ("storage-poll".equals(pollType)) { - currentResult = executeStorageCheck(triggerConfig); - triggerType = AutomationConstants.TRIGGER_STORAGE_POLL; - } else if ("db-poll".equals(pollType)) { - currentResult = executeDbCheck(triggerConfig); - triggerType = AutomationConstants.TRIGGER_DB_POLL; - } else { - throw new IllegalArgumentException("Unknown poll type: " + pollType + ". Expected storage-poll or db-poll"); - } - - if (currentResult == null) { - return noChange("Check pixel returned no result"); - } - - String currentHash = sha256(currentResult); - Map state = loadPollState(projectId); - String previousHash = state.getOrDefault(pollType, ""); - - if (currentHash.equals(previousHash)) { - classLogger.debug("Poll trigger for project {} ({}): no change detected", projectId, pollType); - return noChange("No change detected"); - } - - // State changed — fire automation - classLogger.info("Poll trigger for project {} ({}): change detected, firing automation", projectId, pollType); - state.put(pollType, currentHash); - savePollState(projectId, state); - - // Fire TriggerAutomation with the appropriate trigger type - String pixel = "TriggerAutomation(project=[\"" + projectId + "\"], triggerType=[\"" + triggerType + "\"]);"; - try { - this.insight.runPixel(pixel); - } catch (Exception e) { - classLogger.error("Failed to fire automation for project {} after change detected: {}", projectId, e.getMessage(), e); - throw new IllegalStateException("Change detected but automation trigger failed: " + e.getMessage(), e); - } - - Map result = new LinkedHashMap<>(); - result.put("triggered", true); - result.put("pollType", pollType); - result.put("projectId", projectId); - return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); - } - - @SuppressWarnings("unchecked") - private Map loadTriggerConfig(String projectId) { - try { - String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); - File automationFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); - if (!automationFile.exists()) return null; - - String json = Files.readString(automationFile.toPath(), StandardCharsets.UTF_8); - Map doc = GSON.fromJson(json, new TypeToken>(){}.getType()); - Map graph = (Map) doc.get("graph"); - if (graph == null) return null; - List> nodes = (List>) graph.get("nodes"); - if (nodes == null) return null; - - for (Map node : nodes) { - if (AutomationConstants.NODE_TRIGGER.equals(node.get("type"))) { - Object cfg = node.get("config"); - if (cfg instanceof Map) return (Map) cfg; - } - } - } catch (Exception e) { - classLogger.warn("Failed to load trigger config for {}: {}", projectId, e.getMessage()); - } - return null; - } - - private String executeStorageCheck(Map triggerConfig) { - String engineId = str(triggerConfig.get("storagePollEngineId")); - String path = str(triggerConfig.get("storagePollPath")); - if (engineId == null || path == null) { - classLogger.warn("Storage poll config incomplete: engineId={}, path={}", engineId, path); - return null; - } - String pixel = "ListStoragePath(storage=[\"" + engineId + "\"], storagePath=[\"" + path + "\"]);"; - try { - Object out = this.insight.runPixel(pixel); - return out != null ? GSON.toJson(out) : null; - } catch (Exception e) { - classLogger.warn("Storage poll check failed: {}", e.getMessage()); - return null; - } - } - - private String executeDbCheck(Map triggerConfig) { - String engineId = str(triggerConfig.get("dbPollEngineId")); - String query = str(triggerConfig.get("dbPollQuery")); - if (engineId == null || query == null) { - classLogger.warn("DB poll config incomplete: engineId={}, query={}", engineId, query); - return null; - } - String pixel = "SqlQuery(database=[\"" + engineId + "\"], query=[\"" + query + "\"]);"; - try { - Object out = this.insight.runPixel(pixel); - return out != null ? GSON.toJson(out) : null; - } catch (Exception e) { - classLogger.warn("DB poll check failed: {}", e.getMessage()); - return null; - } - } - - @SuppressWarnings("unchecked") - private Map loadPollState(String projectId) { - try { - String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); - File stateFile = new File(portalsFolder + "/" + POLL_STATE_FILE); - if (!stateFile.exists()) return new HashMap<>(); - String json = Files.readString(stateFile.toPath(), StandardCharsets.UTF_8); - Map state = GSON.fromJson(json, new TypeToken>(){}.getType()); - return state != null ? state : new HashMap<>(); - } catch (Exception e) { - classLogger.warn("Could not load poll state for {}: {}", projectId, e.getMessage()); - return new HashMap<>(); - } - } - - private void savePollState(String projectId, Map state) { - try { - String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); - File stateFile = new File(portalsFolder + "/" + POLL_STATE_FILE); - stateFile.getParentFile().mkdirs(); - Files.writeString(stateFile.toPath(), GSON.toJson(state), StandardCharsets.UTF_8); - } catch (Exception e) { - classLogger.warn("Could not save poll state for {}: {}", projectId, e.getMessage()); - } - } - - private static String sha256(String input) { - try { - MessageDigest md = MessageDigest.getInstance("SHA-256"); - byte[] bytes = md.digest(input.getBytes(StandardCharsets.UTF_8)); - StringBuilder sb = new StringBuilder(); - for (byte b : bytes) sb.append(String.format("%02x", b)); - return sb.toString(); - } catch (Exception e) { - return String.valueOf(input.hashCode()); - } - } - - private static String str(Object v) { - return (v != null && !v.toString().isBlank()) ? v.toString() : null; - } - - private static NounMetadata noChange(String reason) { - Map r = new LinkedHashMap<>(); - r.put("triggered", false); - r.put("reason", reason); - return new NounMetadata(r, PixelDataType.MAP, PixelOperationType.OPERATION); - } -} diff --git a/src/prerna/reactor/automation/GenerateAutomationWebhookSecretReactor.java b/src/prerna/reactor/automation/GenerateAutomationWebhookSecretReactor.java deleted file mode 100644 index def6820a57e..00000000000 --- a/src/prerna/reactor/automation/GenerateAutomationWebhookSecretReactor.java +++ /dev/null @@ -1,152 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.reactor.automation; - -import java.io.File; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.reflect.TypeToken; - -import prerna.auth.AccessToken; -import prerna.auth.AuthProvider; -import prerna.auth.User; -import prerna.auth.utils.SecurityProjectUtils; -import prerna.reactor.AbstractReactor; -import prerna.sablecc2.om.PixelDataType; -import prerna.sablecc2.om.PixelOperationType; -import prerna.sablecc2.om.nounmeta.NounMetadata; -import prerna.util.AssetUtility; - -/** - * Generates (or regenerates) a webhook secret for an automation project. - * - *

Pixel: {@code GenerateAutomationWebhookSecret(project=["projectId"])} - * - *

Stores the secret in {@code automation-config.json} under the key - * {@code WEBHOOK_SECRET} (marked sensitive=true). Returns the plain-text - * secret once — it is not retrievable again from the API. - */ -public class GenerateAutomationWebhookSecretReactor extends AbstractReactor { - - private static final Logger classLogger = LogManager.getLogger(GenerateAutomationWebhookSecretReactor.class); - private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); - - private static final String WEBHOOK_SECRET_KEY = "WEBHOOK_SECRET"; - private static final String WEBHOOK_USER_KEY = "WEBHOOK_USER"; - - public GenerateAutomationWebhookSecretReactor() { - this.keysToGet = new String[]{ "project" }; - } - - @Override - public NounMetadata execute() { - organizeKeys(); - String projectId = this.keyValue.get(this.keysToGet[0]); - if (projectId == null || projectId.isBlank()) { - throw new IllegalArgumentException("Must provide a project id"); - } - - projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); - if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { - throw new IllegalArgumentException("Project does not exist or user does not have edit access"); - } - - String secret = UUID.randomUUID().toString().replace("-", "") + - UUID.randomUUID().toString().replace("-", ""); - - String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); - File configFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_CONFIG_FILE_NAME); - - List> entries = new ArrayList<>(); - if (configFile.exists()) { - try { - String json = Files.readString(configFile.toPath(), StandardCharsets.UTF_8); - List> existing = GSON.fromJson(json, - new TypeToken>>(){}.getType()); - if (existing != null) { - // copy all entries except the existing WEBHOOK_SECRET / WEBHOOK_USER - for (Map e : existing) { - Object key = e.get("key"); - if (!WEBHOOK_SECRET_KEY.equals(key) && !WEBHOOK_USER_KEY.equals(key)) { - entries.add(e); - } - } - } - } catch (Exception e) { - classLogger.warn("Could not parse existing automation config for {}: {}", projectId, e.getMessage()); - } - } - - // Build "PROVIDER:id,PROVIDER:id" string for the executing user - User callingUser = this.insight.getUser(); - StringBuilder userAccessBuilder = new StringBuilder(); - for (AuthProvider provider : callingUser.getLogins()) { - AccessToken token = callingUser.getAccessToken(provider); - if (token != null) { - if (userAccessBuilder.length() > 0) userAccessBuilder.append(","); - userAccessBuilder.append(provider.name()).append(":").append(token.getId()); - } - } - - Map secretEntry = new LinkedHashMap<>(); - secretEntry.put("key", WEBHOOK_SECRET_KEY); - secretEntry.put("value", secret); - secretEntry.put("sensitive", true); - entries.add(secretEntry); - - Map userEntry = new LinkedHashMap<>(); - userEntry.put("key", WEBHOOK_USER_KEY); - userEntry.put("value", userAccessBuilder.toString()); - userEntry.put("sensitive", true); - entries.add(userEntry); - - try { - configFile.getParentFile().mkdirs(); - Files.writeString(configFile.toPath(), GSON.toJson(entries), StandardCharsets.UTF_8); - } catch (Exception e) { - throw new IllegalStateException("Failed to save webhook secret: " + e.getMessage(), e); - } - - Map result = new LinkedHashMap<>(); - result.put("secret", secret); - result.put("projectId", projectId); - result.put("note", "Store this secret securely - it cannot be retrieved again. Pass it in the X-Webhook-Secret header when calling the webhook endpoint."); - return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); - } -} diff --git a/src/prerna/reactor/automation/GetAutomationRunReactor.java b/src/prerna/reactor/automation/GetAutomationRunReactor.java index 7d5c5db3465..769b7252320 100644 --- a/src/prerna/reactor/automation/GetAutomationRunReactor.java +++ b/src/prerna/reactor/automation/GetAutomationRunReactor.java @@ -32,8 +32,6 @@ import java.util.List; import java.util.Map; -import com.google.gson.Gson; - import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -79,10 +77,8 @@ public NounMetadata execute() { return new NounMetadata(notFound, PixelDataType.MAP, PixelOperationType.OPERATION); } - // Build node results with for-each progress and while-loop iteration data List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(runId); List> nodeResults = new ArrayList<>(); - Gson gson = new Gson(); for (Map nodeOutput : nodeOutputs) { Map nodeResult = new HashMap<>(); @@ -92,36 +88,6 @@ public NounMetadata execute() { nodeResult.put(AutomationConstants.DURATION_MS, nodeOutput.get(AutomationConstants.DURATION_MS)); nodeResult.put(AutomationConstants.OUTPUT_PREVIEW, nodeOutput.get(AutomationConstants.OUTPUT_PREVIEW)); nodeResult.put(AutomationConstants.ERROR_MESSAGE, nodeOutput.get(AutomationConstants.ERROR_MESSAGE)); - - // Include for-each progress if this node has a row count - Object rowCount = nodeOutput.get(AutomationConstants.ROW_COUNT); - if (rowCount != null) { - nodeResult.put(AutomationConstants.ROW_COUNT, rowCount); - String nodeId = (String) nodeOutput.get(AutomationConstants.NODE_ID); - Map progress = AutomationDatabaseUtility.getForEachProgress(runId, nodeId); - if (!progress.isEmpty()) { - nodeResult.put("forEachProgress", progress); - } - } - - // Parse while-loop iteration data stored in OUTPUT_VALUE - Object outputValue = nodeOutput.get(AutomationConstants.OUTPUT_VALUE); - if (outputValue instanceof String) { - String outputStr = (String) outputValue; - if (outputStr.contains("\"__whileResult\":true")) { - try { - @SuppressWarnings("unchecked") - Map wr = gson.fromJson(outputStr, Map.class); - Object iterations = wr.get("iterations"); - if (iterations != null) { - nodeResult.put("iterationResults", iterations); - } - } catch (Exception ignored) { - // malformed JSON - skip - } - } - } - nodeResults.add(nodeResult); } diff --git a/src/prerna/reactor/automation/PixelExecutionUtils.java b/src/prerna/reactor/automation/PixelExecutionUtils.java deleted file mode 100644 index fd4b7ca83cf..00000000000 --- a/src/prerna/reactor/automation/PixelExecutionUtils.java +++ /dev/null @@ -1,232 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.reactor.automation; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import prerna.om.Insight; -import prerna.om.ThreadStore; -import prerna.sablecc2.om.PixelOperationType; -import prerna.sablecc2.om.nounmeta.NounMetadata; -import prerna.sablecc2.om.task.ITask; - -/** - * Shared utility for executing Pixel expressions and returning clean, serializable results. - * - *

Handles the common pitfalls of raw {@code insight.runPixel()} calls: - *

    - *
  • {@link ITask} materialization - query reactors return lazy cursors that must be collected
    • - *
  • Timeout enforcement - prevents hung queries from blocking pipelines indefinitely
    • - *
  • Error extraction - detects {@link prerna.sablecc2.om.PixelOperationType#ERROR} in results
    • - *
  • Null-safe return - always returns a usable value
    • - *
- */ -public final class PixelExecutionUtils { - - private static final Logger classLogger = LogManager.getLogger(PixelExecutionUtils.class); - - private PixelExecutionUtils() {} - - /** - * Executes a pixel expression and returns a clean, serializable result. - * - * @param insight the execution context (carries user, varstore, etc.) - * @param pixel the fully resolved pixel string to execute - * @param timeoutSeconds max execution time; 0 or negative means no timeout - * @return the pixel result as a serializable object (Map, List, String, Number, etc.), or null - * @throws AutomationNodeTimeoutException if execution exceeds the timeout - * @throws AutomationPixelException if the pixel returns an error result - */ - public static Object runAndCollect(Insight insight, String pixel, int timeoutSeconds) { - if (pixel == null || pixel.isBlank()) { - return null; - } - - classLogger.debug("Executing pixel (timeout={}s): {}", timeoutSeconds, - pixel.length() > 200 ? pixel.substring(0, 200) + "..." : pixel); - - NounMetadata result; - if (timeoutSeconds > 0) { - result = executeWithTimeout(insight, pixel, timeoutSeconds); - } else { - result = executeDirectly(insight, pixel); - } - - if (result == null) { - return null; - } - - checkForError(result, pixel); - return materializeValue(result); - } - - /** Overload with default timeout from {@link AutomationConstants#DEFAULT_TIMEOUT_SECONDS}. */ - public static Object runAndCollect(Insight insight, String pixel) { - return runAndCollect(insight, pixel, AutomationConstants.DEFAULT_TIMEOUT_SECONDS); - } - - /** Serializes a pixel result to a JSON string for DB storage. */ - public static String serializeResult(Object result) { - if (result == null) return ""; - if (result instanceof String) return (String) result; - return AutomationExecutionUtils.GSON.toJson(result); - } - - /** - * Generates a truncated preview string for quick UI display. - * Returns null if input is null. - */ - public static String generatePreview(String serializedOutput) { - if (serializedOutput == null) return null; - int maxLength = AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH; - return serializedOutput.length() <= maxLength - ? serializedOutput - : serializedOutput.substring(0, maxLength); - } - - // -- Private implementation ---------------------------------------------------- - - private static NounMetadata executeWithTimeout(Insight insight, String pixel, int timeoutSeconds) { - // A new executor is created per timed call and shut down immediately after — no leak. - ExecutorService executor = Executors.newSingleThreadExecutor(r -> { - Thread t = new Thread(r, "automation-pixel-exec"); - t.setDaemon(true); - return t; - }); - - // ThreadStore is a plain ThreadLocal — not inherited by pool threads. - // Snapshot the caller's context so the worker thread has user/session/insight access. - Map callerContext = ThreadStore.getTheadMapObject(); - final Map contextSnapshot = - callerContext != null ? new HashMap<>(callerContext) : null; - - try { - Callable task = () -> { - if (contextSnapshot != null && !contextSnapshot.isEmpty()) { - ThreadStore.getInsightId(); - ThreadStore.setThreadMapObject(contextSnapshot); - } - try { - return executeDirectly(insight, pixel); - } finally { - ThreadStore.remove(); - } - }; - - Future future = executor.submit(task); - try { - return future.get(timeoutSeconds, TimeUnit.SECONDS); - } catch (TimeoutException e) { - future.cancel(true); - throw new AutomationNodeTimeoutException(pixel, timeoutSeconds); - } catch (ExecutionException e) { - Throwable cause = e.getCause(); - if (cause instanceof RuntimeException) throw (RuntimeException) cause; - throw new IllegalStateException("Pixel execution failed: " + cause.getMessage(), cause); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new IllegalStateException("Pixel execution interrupted", e); - } - } finally { - executor.shutdownNow(); - } - } - - private static NounMetadata executeDirectly(Insight insight, String pixel) { - List results = insight.runPixel(pixel).getResults(); - if (results == null || results.isEmpty()) return null; - // For multi-statement pixels the meaningful result is the last one. - return results.get(results.size() - 1); - } - - private static void checkForError(NounMetadata result, String pixel) { - if (result.getOpType() != null && result.getOpType().contains(PixelOperationType.ERROR)) { - String errorMsg = result.getValue() != null ? result.getValue().toString() : "Unknown pixel error"; - throw new AutomationPixelException(pixel, errorMsg); - } - } - - private static Object materializeValue(NounMetadata result) { - Object value = result.getValue(); - if (value == null) return null; - - if (value instanceof ITask) { - classLogger.debug("Materializing ITask result"); - try { - return ((ITask) value).collect(false); - } catch (Exception e) { - classLogger.error("Failed to materialize ITask: {}", e.getMessage(), e); - throw new IllegalStateException("Failed to materialize query result: " + e.getMessage(), e); - } - } - return value; - } - - // -- Exception types ----------------------------------------------------------- - - /** Thrown when a pixel execution exceeds its configured timeout. */ - public static class AutomationNodeTimeoutException extends RuntimeException { - private final int timeoutSeconds; - - public AutomationNodeTimeoutException(String pixel, int timeoutSeconds) { - super("Pixel execution timed out after " + timeoutSeconds + " seconds: " + - (pixel.length() > 100 ? pixel.substring(0, 100) + "..." : pixel)); - this.timeoutSeconds = timeoutSeconds; - } - - public int getTimeoutSeconds() { - return timeoutSeconds; - } - } - - /** Thrown when a pixel returns an ERROR operation type. */ - public static class AutomationPixelException extends RuntimeException { - private final String pixel; - - public AutomationPixelException(String pixel, String errorMessage) { - super(errorMessage); - this.pixel = pixel; - } - - public String getPixel() { - return pixel; - } - } -} diff --git a/src/prerna/reactor/automation/ResumeAutomationRunReactor.java b/src/prerna/reactor/automation/ResumeAutomationRunReactor.java deleted file mode 100644 index 1bcc3649136..00000000000 --- a/src/prerna/reactor/automation/ResumeAutomationRunReactor.java +++ /dev/null @@ -1,113 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.reactor.automation; - -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import prerna.auth.utils.SecurityProjectUtils; -import prerna.reactor.AbstractReactor; -import prerna.sablecc2.om.PixelDataType; -import prerna.sablecc2.om.PixelOperationType; -import prerna.sablecc2.om.nounmeta.NounMetadata; - -/** - * Resumes a failed or interrupted automation run from the first failed node. - * - *

Pixel: {@code ResumeAutomationRun(project=["appId"], runId=["failed-run-id"])} - * - *

Validates the target run exists and is in FAILED or INTERRUPTED status, - * then delegates to {@link TriggerAutomationReactor} with the {@code resumeRunId} - * parameter set. This creates a new run that skips previously successful nodes - * and re-executes from the failure point. - */ -public class ResumeAutomationRunReactor extends AbstractReactor { - - private static final Logger classLogger = LogManager.getLogger(ResumeAutomationRunReactor.class); - - public ResumeAutomationRunReactor() { - this.keysToGet = new String[]{ "project", "runId" }; - this.keyRequired = new int[]{ 1, 1 }; - } - - @Override - public NounMetadata execute() { - organizeKeys(); - String projectId = this.keyValue.get(this.keysToGet[0]); - String runId = this.keyValue.get(this.keysToGet[1]); - - if (projectId == null || projectId.isEmpty()) { - throw new IllegalArgumentException("Must provide a project id"); - } - if (runId == null || runId.isEmpty()) { - throw new IllegalArgumentException("Must provide the run id to resume"); - } - - // Auth check - projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); - if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { - throw new IllegalArgumentException("Project does not exist or user does not have access"); - } - - // Validate the run exists and is resumable - Map runDetail = AutomationDatabaseUtility.getRunDetail(runId); - if (runDetail == null) { - throw new IllegalArgumentException("Run not found: " + runId); - } - - String status = (String) runDetail.get(AutomationConstants.STATUS); - if (!AutomationConstants.STATUS_FAILED.equals(status) - && !AutomationConstants.STATUS_INTERRUPTED.equals(status)) { - throw new IllegalArgumentException( - "Can only resume FAILED or INTERRUPTED runs. Current status: " + status); - } - - // Verify the run belongs to this project - String runProjectId = (String) runDetail.get(AutomationConstants.PROJECT_ID); - if (!projectId.equals(runProjectId)) { - throw new IllegalArgumentException("Run " + runId + " does not belong to project " + projectId); - } - - classLogger.info("Resuming automation run {} for project {}", runId, projectId); - - // Both values come from validated/DB sources (projectId from testUserProjectIdForAlias, - // runId from AUTOMATION_RUNS), so injection is not expected - guard defensively. - if (projectId.contains("\"") || projectId.contains("]") || - runId.contains("\"") || runId.contains("]")) { - throw new IllegalArgumentException("Invalid characters in project ID or run ID"); - } - - String pixel = "TriggerAutomation(project=[\"" + projectId + "\"], " - + "manual=[\"true\"], resumeRunId=[\"" + runId + "\"]);"; - return new NounMetadata( - PixelExecutionUtils.runAndCollect(this.insight, pixel, 0), - PixelDataType.MAP, PixelOperationType.OPERATION); - } -} diff --git a/src/prerna/reactor/automation/RunAutomationNodeReactor.java b/src/prerna/reactor/automation/RunAutomationNodeReactor.java index fbffe2c4a01..fa15906aa13 100644 --- a/src/prerna/reactor/automation/RunAutomationNodeReactor.java +++ b/src/prerna/reactor/automation/RunAutomationNodeReactor.java @@ -35,6 +35,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -45,22 +46,19 @@ import prerna.auth.utils.SecurityProjectUtils; import prerna.reactor.AbstractReactor; +import prerna.reactor.automation.nodes.AutomationNodeContext; +import prerna.reactor.automation.nodes.AutomationNodeExecutors; +import prerna.reactor.automation.nodes.IAutomationNodeExecutor; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; import prerna.sablecc2.om.nounmeta.NounMetadata; import prerna.util.AssetUtility; /** - * Executes a single automation node for testing purposes. + * Executes a single automation node for testing/preview purposes. + * Result is NOT persisted to any run record. * - *

Pixel: {@code RunAutomationNode(project=["appId"], nodeId=["node-id"], runId=["optional-context-run"])} - * - *

Loads the automation definition, finds the target node, optionally loads scope from a - * prior run's outputs, and executes just that one node. The result is NOT persisted to - * any run - this is a test/preview operation. - * - *

When {@code runId} is provided, prior node outputs from that run are loaded into - * the scope so that {@code ${varName}} references resolve correctly. + * Pixel: {@code RunAutomationNode(project=["appId"], nodeId=["node-id"], runId=["optional-context-run"])} */ public class RunAutomationNodeReactor extends AbstractReactor { @@ -86,36 +84,49 @@ public NounMetadata execute() { throw new IllegalArgumentException("Must provide a node id"); } - // Auth check projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { throw new IllegalArgumentException("Project does not exist or user does not have access"); } - // Load automation and find the target node Map node = findNode(projectId, nodeId); if (node == null) { throw new IllegalArgumentException("Node not found in automation: " + nodeId); } - // Build scope from context run (if provided) Map scope = buildScope(contextRunId); Map configMap = AutomationExecutionUtils.loadConfig(projectId); - // Execute the node long startMs = System.currentTimeMillis(); try { - Object rawOutput = executeNodePixel(node, scope, configMap); + String type = (String) node.get("type"); + Object rawOutput; + + if (AutomationConstants.NODE_TRIGGER.equals(type)) { + rawOutput = scope.get("triggered_at"); + } else { + IAutomationNodeExecutor executor = AutomationNodeExecutors.EXECUTORS.get(type); + if (executor == null) { + throw new IllegalArgumentException("Unsupported node type: " + type); + } + AutomationNodeContext ctx = new AutomationNodeContext( + "test", projectId, node, scope, configMap, + this.insight, new AtomicBoolean(false)); + rawOutput = executor.execute(ctx); + } + @SuppressWarnings("unchecked") Map transformConfig = (Map) node.get("outputTransform"); String transformed = AutomationExecutionUtils.applyOutputTransform(rawOutput, transformConfig); long durationMs = System.currentTimeMillis() - startMs; + String preview = (transformed != null && transformed.length() > AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH) + ? transformed.substring(0, AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH) : transformed; Map result = new HashMap<>(); result.put(AutomationConstants.NODE_ID, nodeId); result.put(AutomationConstants.STATUS, AutomationConstants.NODE_STATUS_SUCCESS); result.put(AutomationConstants.DURATION_MS, durationMs); - result.put(AutomationConstants.OUTPUT_PREVIEW, PixelExecutionUtils.generatePreview(transformed)); + result.put(AutomationConstants.OUTPUT_PREVIEW, preview); result.put(AutomationConstants.OUTPUT_VALUE, transformed); return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); @@ -132,8 +143,6 @@ public NounMetadata execute() { } } - // -- Helpers ------------------------------------------------------------------- - @SuppressWarnings("unchecked") private Map findNode(String projectId, String nodeId) { String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); @@ -148,9 +157,7 @@ private Map findNode(String projectId, String nodeId) { List> nodes = (List>) graph.get("nodes"); if (nodes != null) { for (Map node : nodes) { - if (nodeId.equals(node.get("id"))) { - return node; - } + if (nodeId.equals(node.get("id"))) return node; } } } catch (IOException e) { @@ -179,21 +186,4 @@ private Map buildScope(String contextRunId) { } return scope; } - - private Object executeNodePixel(Map node, Map scope, - Map configMap) { - String type = (String) node.get("type"); - if (AutomationConstants.NODE_TRIGGER.equals(type)) { - return scope.get("triggered_at"); - } - - String builtPixel = (String) node.get("builtPixel"); - if (builtPixel == null || builtPixel.isBlank() || builtPixel.startsWith("//")) { - throw new IllegalStateException("Node has no compiled pixel - save the automation first"); - } - - String resolved = AutomationExecutionUtils.resolve(builtPixel, scope, configMap); - return PixelExecutionUtils.runAndCollect(this.insight, resolved, - AutomationConstants.DEFAULT_TIMEOUT_SECONDS); - } } diff --git a/src/prerna/reactor/automation/TriggerAutomationReactor.java b/src/prerna/reactor/automation/TriggerAutomationReactor.java index 8f9583b6ae0..869006eb97d 100644 --- a/src/prerna/reactor/automation/TriggerAutomationReactor.java +++ b/src/prerna/reactor/automation/TriggerAutomationReactor.java @@ -34,10 +34,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -56,55 +54,19 @@ import prerna.om.ThreadStore; import prerna.reactor.AbstractReactor; import prerna.reactor.automation.nodes.AutomationNodeContext; -import prerna.reactor.automation.nodes.ChildAutomationRunner; -import prerna.reactor.automation.nodes.DatabaseEngineNodeExecutor; -import prerna.reactor.automation.nodes.FunctionEngineNodeExecutor; +import prerna.reactor.automation.nodes.AutomationNodeExecutors; import prerna.reactor.automation.nodes.IAutomationNodeExecutor; -import prerna.reactor.automation.nodes.ModelEngineNodeExecutor; -import prerna.reactor.automation.nodes.NodeDispatcher; -import prerna.reactor.automation.nodes.PixelNodeExecutor; -import prerna.reactor.automation.nodes.StorageEngineNodeExecutor; -import prerna.reactor.automation.nodes.VectorEngineNodeExecutor; -import prerna.reactor.automation.nodes.WaitNodeExecutor; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; import prerna.sablecc2.om.nounmeta.NounMetadata; import prerna.util.Utility; -/** - * Executes an automation app's graph top-to-bottom with DB-backed state. - * - *

Pixel: {@code TriggerAutomation(project=["appId"], manual=["true"])} - *

Pixel: {@code TriggerAutomation(project=["appId"], resumeRunId=["uuid"])} - * - *

Execution model: - *

    - *
  • Concurrency guard - rejects if a run is already active for this project
    • - *
  • DB checkpoint per node - each completed node is committed immediately
    • - *
  • Stop on error - first node failure halts the pipeline
    • - *
  • Heartbeat - updated every 30s to prove liveness
    • - *
  • Resume - skips nodes that succeeded in a prior run, re-runs from failure
    • - *
- * - *

State is written to AUTOMATION_RUNS and AUTOMATION_NODE_OUTPUTS in the scheduler DB - * via {@link AutomationDatabaseUtility}. - */ public class TriggerAutomationReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(TriggerAutomationReactor.class); - /** - * Registry of active run cancellation flags. Keyed by runId. - * When a cancel is requested, the flag is set to true and the executor checks - * between nodes. - */ private static final ConcurrentHashMap CANCELLATION_FLAGS = new ConcurrentHashMap<>(); - /** - * Background pool for automation execution. Bounded at 20 concurrent runs with a small queue - * for brief spikes. Rejects beyond capacity so the caller gets an immediate error rather than - * unbounded thread growth. - */ private static final ExecutorService AUTOMATION_EXECUTOR = new ThreadPoolExecutor( 2, 20, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(10), @@ -116,53 +78,18 @@ public class TriggerAutomationReactor extends AbstractReactor { new ThreadPoolExecutor.AbortPolicy() ); - /** - * Registry mapping a node's {@code type} to the executor that runs it - replaces the - * previous if/else chain in {@link #executeSingleNode}. Mirrors the existing SEMOSS pattern - * for "one operation, many type-specific implementations, resolved by a type key" - * (see {@code IModelEngine} -> {@code Utility.getModel(engineId)}, {@code IMCP}). - * Executors are stateless and shared across every run/node. - * - * Phase 1 executors only. Phase 2 (conditional, switch, email, http, set-variable, transform, - * retry, try-catch) and Phase 3 (for-each, while-loop, parallel, sub-automation) entries are - * added in their respective bring-over phases. - */ - private static final Map EXECUTORS = Map.of( - AutomationConstants.NODE_WAIT, new WaitNodeExecutor(), - AutomationConstants.NODE_DATABASE_ENGINE, new DatabaseEngineNodeExecutor(), - AutomationConstants.NODE_MODEL_ENGINE, new ModelEngineNodeExecutor(), - AutomationConstants.NODE_VECTOR_ENGINE, new VectorEngineNodeExecutor(), - AutomationConstants.NODE_STORAGE_ENGINE, new StorageEngineNodeExecutor(), - AutomationConstants.NODE_FUNCTION_ENGINE, new FunctionEngineNodeExecutor() - ); - - /** - * Default executor for node types with no dedicated entry above - {@code trigger}, - * {@code app}, and {@code custom-pixel} - which are genuinely arbitrary/composed Pixel with - * no single backing engine. See {@link PixelNodeExecutor}. - */ - private static final IAutomationNodeExecutor PIXEL_EXECUTOR = new PixelNodeExecutor(); - public TriggerAutomationReactor() { - this.keysToGet = new String[]{ "project", "manual", "resumeRunId", "triggerType" }; - this.keyRequired = new int[]{ 1, 0, 0, 0 }; + this.keysToGet = new String[]{ "project" }; + this.keyRequired = new int[]{ 1 }; } @Override public NounMetadata execute() { organizeKeys(); String projectId = getProjectId(); - String resumeRunId = this.keyValue.get(this.keysToGet[2]); - - // Determine trigger type - String triggerType = determineTriggerType(resumeRunId); String userId = getUserId(); String runId = UUID.randomUUID().toString(); - // Concurrency guard - atomic claim against the shared scheduler DB, so this is correct - // across every pod in a cluster, not just within this JVM. Prevents two concurrent - // triggers for the same project from both starting a run (which would double up any - // node with side effects, e.g. a database-update node running twice). if (!AutomationDatabaseUtility.claimActiveRun(projectId, runId)) { String activeRun = AutomationDatabaseUtility.getActiveRun(projectId); throw new IllegalArgumentException( @@ -171,7 +98,6 @@ public NounMetadata execute() { } try { - // Load automation definition and config Map doc = AutomationExecutionUtils.loadAutomationDoc(projectId); @SuppressWarnings("unchecked") Map graph = (Map) doc.get("graph"); @@ -181,28 +107,15 @@ public NounMetadata execute() { List> edges = (List>) graph.get("edges"); Map configMap = AutomationExecutionUtils.loadConfig(projectId); - // Topological sort List> ordered = AutomationExecutionUtils.topoSort(nodes, edges); if (ordered.isEmpty()) { throw new IllegalArgumentException("Automation has no nodes to execute"); } - // Create run record in DB AutomationDatabaseUtility.insertRun(runId, projectId, AutomationConstants.DEFAULT_AUTOMATION_ID, - triggerType, resumeRunId, ordered.size(), userId); + AutomationConstants.TRIGGER_MANUAL, ordered.size(), userId); AutomationDatabaseUtility.insertAllNodeOutputs(runId, ordered); - // Load prior outputs if resuming - Map priorOutputs = loadPriorOutputs(resumeRunId); - - // Execute nodes on a background thread - an automation can run for hours (large for-each - // ingestion jobs), so it must never block the calling request/websocket thread. - // Progress is checkpointed to AUTOMATION_RUNS/AUTOMATION_NODE_OUTPUTS per node; the - // caller (FE) polls GetAutomationRun(runId) for live status instead of awaiting this. - // Capture the calling thread's ThreadStore (user, session, insight id, scheduler mode) - // so it can be re-seeded on the background executor thread. ThreadStore is a plain - // ThreadLocal and is NOT inherited by pool threads; without this, reactors that read - // ThreadStore during node execution would see null context. Map parentContext = ThreadStore.getTheadMapObject(); final Map contextSnapshot = parentContext != null ? new HashMap<>(parentContext) : null; @@ -211,10 +124,7 @@ public NounMetadata execute() { AUTOMATION_EXECUTOR.submit(() -> { installThreadContext(contextSnapshot); try { - // executeNodes' own finally always releases the active-run slot - - // including when it throws, which is caught here - so no explicit - // release is needed in this catch block. - executeNodes(runId, projectId, ordered, configMap, priorOutputs); + executeNodes(runId, projectId, ordered, configMap); } catch (Exception e) { classLogger.error("Unhandled error executing automation run {}: {}", runId, e.getMessage(), e); AutomationDatabaseUtility.updateRunStatus(runId, @@ -224,8 +134,6 @@ public NounMetadata execute() { } }); } catch (RejectedExecutionException e) { - // Never submitted - executeNodes' own finally (which normally releases the - // active-run slot) will never run. Release happens in the outer catch below. AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_FAILED, null, "Server is at capacity - too many concurrent automation runs"); throw new IllegalStateException("Too many concurrent automation runs. Please try again shortly."); @@ -235,9 +143,6 @@ public NounMetadata execute() { ordered.size(), 0, null, new ArrayList<>()); return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); } catch (RuntimeException e) { - // Any failure before (or in lieu of) the run being successfully handed off to the - // background executor means executeNodes' own finally will never run to release the - // slot - release it here so the project isn't left permanently blocked. AutomationDatabaseUtility.releaseActiveRun(projectId, runId); throw e; } @@ -246,51 +151,24 @@ public NounMetadata execute() { // -- Core Execution ------------------------------------------------------------ private Map executeNodes(String runId, String projectId, - List> ordered, Map configMap, - Map priorOutputs) { - return executeNodes(runId, projectId, ordered, configMap, priorOutputs, - null, Collections.singleton(projectId)); - } - - /** - * Executes an ordered node list for a run. Used both for top-level runs (manual/scheduled/ - * resume, {@code extraInitialScope} null) and for sub-automation calls, where - * {@code extraInitialScope} carries the resolved {@code inputMapping} values and - * {@code ancestorProjectIds} carries the chain of project ids already executing on this - * call stack (self/transitive-call cycle guard). - */ - private Map executeNodes(String runId, String projectId, - List> ordered, Map configMap, - Map priorOutputs, Map extraInitialScope, - Set ancestorProjectIds) { + List> ordered, Map configMap) { - // Register cancellation flag AtomicBoolean cancelled = new AtomicBoolean(false); CANCELLATION_FLAGS.put(runId, cancelled); - // Start heartbeat ScheduledExecutorService heartbeat = startHeartbeat(runId); Map scope = buildInitialScope(runId); - if (extraInitialScope != null) { - scope.putAll(extraInitialScope); - } List> nodeResults = new ArrayList<>(); int completedCount = 0; try { - for (int i = 0; i < ordered.size(); i++) { - Map node = ordered.get(i); + for (Map node : ordered) { String nodeId = (String) node.get("id"); String nodeLabel = (String) node.get("label"); String outputVar = (String) node.get("outputVar"); String nodeType = (String) node.get("type"); - // Check cancellation between nodes - the local AtomicBoolean is a same-pod fast - // path (set instantly by CancelAutomationRunReactor when it lands on this pod); - // isCancelRequested() is the cluster-safe source of truth, so a cancel request - // that landed on a different pod than the one executing this run is still - // honored here. if (cancelled.get() || AutomationDatabaseUtility.isCancelRequested(runId)) { AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_CANCELLED, nodeId, "Run cancelled by user"); @@ -300,23 +178,9 @@ private Map executeNodes(String runId, String projectId, ordered.size(), completedCount, nodeId, nodeResults); } - // Resume: skip nodes that already succeeded in the prior run - if (shouldSkipForResume(nodeId, outputVar, priorOutputs, scope)) { - nodeResults.add(buildNodeResult(nodeId, nodeLabel, - AutomationConstants.NODE_STATUS_SKIPPED, 0, - PixelExecutionUtils.generatePreview(priorOutputs.get(nodeId)), null)); - completedCount++; - AutomationDatabaseUtility.updateHeartbeat(runId, completedCount); - continue; - } - - // Execute this node - catching AutomationCancelledException separately so - // mid-node cancellations (e.g. WaitNodeExecutor interrupted mid-sleep) produce - // CANCELLED run status instead of FAILED. Map nodeResult; try { - nodeResult = executeSingleNode( - runId, projectId, node, scope, configMap, completedCount, ancestorProjectIds); + nodeResult = executeSingleNode(runId, projectId, node, scope, configMap); } catch (AutomationCancelledException ace) { AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_CANCELLED, nodeId, ace.getMessage()); @@ -330,19 +194,14 @@ private Map executeNodes(String runId, String projectId, nodeResults.add(nodeResult); if (AutomationConstants.NODE_STATUS_SUCCESS.equals(status)) { - // Store output in scope for downstream nodes. - // set-variable nodes write individual variables directly into scope - // inside SetVariableNodeExecutor - skip the generic put to avoid - // overwriting those keys with the JSON blob. if (outputVar != null && !outputVar.isEmpty() - && !AutomationConstants.NODE_SET_VARIABLE.equals(nodeType)) { + && !AutomationConstants.NODE_TRIGGER.equals(nodeType)) { String outputValue = (String) nodeResult.get("outputValue"); scope.put(outputVar, outputValue != null ? outputValue : ""); } completedCount++; AutomationDatabaseUtility.updateHeartbeat(runId, completedCount); } else { - // STOP on error String errorMsg = (String) nodeResult.get(AutomationConstants.ERROR_MESSAGE); AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_FAILED, nodeId, errorMsg); @@ -351,138 +210,74 @@ private Map executeNodes(String runId, String projectId, } } - // All nodes succeeded - AutomationDatabaseUtility.updateRunStatus(runId, - AutomationConstants.STATUS_SUCCESS, null, null); + AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_SUCCESS, null, null); return buildRunResult(runId, projectId, AutomationConstants.STATUS_SUCCESS, ordered.size(), completedCount, null, nodeResults); } finally { heartbeat.shutdownNow(); CANCELLATION_FLAGS.remove(runId); - // Release the cluster-safe active-run slot claimed in execute() (top-level runs) or - // SubAutomationNodeExecutor (sub-automation runs) - covers every terminal path - // (success, failure, cancellation) since they all return through here. AutomationDatabaseUtility.releaseActiveRun(projectId, runId); } } private Map executeSingleNode(String runId, String projectId, Map node, - Map scope, Map configMap, int completedCount, - Set ancestorProjectIds) { + Map scope, Map configMap) { String nodeId = (String) node.get("id"); String nodeLabel = (String) node.get("label"); String outputVar = (String) node.get("outputVar"); String type = (String) node.get("type"); - // Mark node as running + // Trigger node is a metadata-only node — just return success + if (AutomationConstants.NODE_TRIGGER.equals(type)) { + return buildNodeResult(nodeId, nodeLabel, AutomationConstants.NODE_STATUS_SUCCESS, 0, + scope.get("triggered_at"), null); + } + AutomationDatabaseUtility.markNodeRunning(runId, nodeId); Timestamp startedAt = toTimestamp(Instant.now()); long startMs = System.currentTimeMillis(); try { - // NodeDispatcher/ChildAutomationRunner recursion callbacks - only composite executors - // (conditional/while-loop/try-catch/switch/retry/parallel) and SubAutomationNodeExecutor - // actually invoke these; every other executor ignores them. - NodeDispatcher nodeDispatcher = (innerNode, innerScope) -> - executeSingleNode(runId, projectId, innerNode, innerScope, configMap, 0, ancestorProjectIds); - ChildAutomationRunner childAutomationRunner = this::executeNodes; AtomicBoolean cancelFlag = CANCELLATION_FLAGS.get(runId); + AutomationNodeContext ctx = new AutomationNodeContext( + runId, projectId, node, scope, configMap, this.insight, cancelFlag); - AutomationNodeContext ctx = new AutomationNodeContext(runId, projectId, node, scope, configMap, - ancestorProjectIds, this.insight, cancelFlag, nodeDispatcher, childAutomationRunner); - - IAutomationNodeExecutor executor = EXECUTORS.getOrDefault(type, PIXEL_EXECUTOR); + IAutomationNodeExecutor executor = AutomationNodeExecutors.EXECUTORS.get(type); + if (executor == null) { + throw new IllegalArgumentException("Unsupported node type: " + type); + } Object rawOutput = executor.execute(ctx); - // ForEachNodeExecutor is the one node type whose result map carries a row count for - // the node checkpoint - detected generically by shape (does the returned Map have a - // "totalRows" entry), not by branching on node type. - Integer rowCount = (rawOutput instanceof Map rawMap && rawMap.get("totalRows") instanceof Integer count) - ? count : null; - - // WhileLoopNodeExecutor similarly marks its per-iteration-history result with a - // "__whileResult" entry on the raw (pre-transform) Map, rather than the caller - // string-sniffing already-serialized JSON for a magic key. - Long whileIterationCount = (rawOutput instanceof Map whileMap - && Boolean.TRUE.equals(whileMap.get("__whileResult")) - && whileMap.get("iterationCount") instanceof Number n) - ? n.longValue() : null; - @SuppressWarnings("unchecked") Map transformConfig = (Map) node.get("outputTransform"); String transformed = AutomationExecutionUtils.applyOutputTransform(rawOutput, transformConfig); long durationMs = System.currentTimeMillis() - startMs; - String preview = whileIterationCount != null - ? whileIterationCount + " iteration" + (whileIterationCount == 1 ? "" : "s") - : PixelExecutionUtils.generatePreview(transformed); + String preview = generatePreview(transformed); - AutomationDatabaseUtility.updateNodeSuccess(runId, nodeId, startedAt, - durationMs, outputVar, transformed, preview, rowCount); + AutomationDatabaseUtility.updateNodeSuccess(runId, nodeId, startedAt, durationMs, outputVar, transformed, preview); Map result = buildNodeResult(nodeId, nodeLabel, AutomationConstants.NODE_STATUS_SUCCESS, durationMs, preview, null); result.put("outputValue", transformed); - if (rowCount != null) { - result.put(AutomationConstants.ROW_COUNT, rowCount); - } return result; } catch (AutomationCancelledException ace) { long durationMs = System.currentTimeMillis() - startMs; - // Mid-node cancellation (e.g. WaitNodeExecutor interrupted mid-sleep). - // Update node as failed since it didn't complete, then propagate so executeNodes - // records the run as CANCELLED rather than FAILED. AutomationDatabaseUtility.updateNodeFailed(runId, nodeId, startedAt, durationMs, ace.getMessage()); throw ace; } catch (Exception e) { long durationMs = System.currentTimeMillis() - startMs; String errorMsg = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); classLogger.error("Node {} ({}) failed: {}", nodeId, nodeLabel, errorMsg, e); - AutomationDatabaseUtility.updateNodeFailed(runId, nodeId, startedAt, durationMs, errorMsg); - return buildNodeResult(nodeId, nodeLabel, AutomationConstants.NODE_STATUS_FAILED, durationMs, null, errorMsg); } } - // -- Resume Logic -------------------------------------------------------------- - - private Map loadPriorOutputs(String resumeRunId) { - Map outputs = new HashMap<>(); - if (resumeRunId == null || resumeRunId.isEmpty()) { - return outputs; - } - - List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(resumeRunId); - for (Map nodeOutput : nodeOutputs) { - String status = (String) nodeOutput.get(AutomationConstants.STATUS); - if (AutomationConstants.NODE_STATUS_SUCCESS.equals(status)) { - String nodeId = (String) nodeOutput.get(AutomationConstants.NODE_ID); - String outputValue = nodeOutput.get(AutomationConstants.OUTPUT_VALUE) != null - ? nodeOutput.get(AutomationConstants.OUTPUT_VALUE).toString() : ""; - outputs.put(nodeId, outputValue); - } - } - return outputs; - } - - private boolean shouldSkipForResume(String nodeId, String outputVar, - Map priorOutputs, Map scope) { - if (priorOutputs.isEmpty() || !priorOutputs.containsKey(nodeId)) { - return false; - } - // Copy prior output to scope so downstream nodes can reference it - String priorValue = priorOutputs.get(nodeId); - if (outputVar != null && !outputVar.isEmpty()) { - scope.put(outputVar, priorValue); - } - return true; - } - // -- Heartbeat ----------------------------------------------------------------- private ScheduledExecutorService startHeartbeat(String runId) { @@ -491,8 +286,6 @@ private ScheduledExecutorService startHeartbeat(String runId) { t.setDaemon(true); return t; }); - // Heartbeat fires every 30 seconds - just proves liveness; per-node count updates - // happen in executeNodes() after each node completes. scheduler.scheduleAtFixedRate(() -> { try { AutomationDatabaseUtility.touchHeartbeat(runId); @@ -506,11 +299,6 @@ private ScheduledExecutorService startHeartbeat(String runId) { // -- Cancellation Support ------------------------------------------------------ - /** - * Requests cancellation of a running automation. Called by CancelAutomationRunReactor. - * Cancellation takes effect between nodes (cannot interrupt mid-pixel), or mid-wait for - * nodes that check the flag during blocking operations (e.g. WaitNodeExecutor). - */ public static boolean requestCancellation(String runId) { AtomicBoolean flag = CANCELLATION_FLAGS.get(runId); if (flag != null) { @@ -520,25 +308,14 @@ public static boolean requestCancellation(String runId) { return false; } - /** - * Seeds the current (background executor) thread's ThreadStore with a snapshot of the - * caller's context. The reading getter forces lazy creation of this thread's map so the - * subsequent putAll has a target. Paired with {@code ThreadStore.remove()} in a finally - * block so pooled threads never leak context between runs. - */ + // -- Helpers ------------------------------------------------------------------- + private static void installThreadContext(Map snapshot) { - if (snapshot == null || snapshot.isEmpty()) { - return; - } - ThreadStore.getInsightId(); // force creation of this thread's ThreadStore map + if (snapshot == null || snapshot.isEmpty()) return; + ThreadStore.getInsightId(); ThreadStore.setThreadMapObject(snapshot); } - // (resolve, getNodeTimeout, applyOutputTransform, strCfg, coerceToMap, loadAutomationDoc, - // topoSort moved to AutomationExecutionUtils) - - // -- Helpers ------------------------------------------------------------------- - private String getProjectId() { String projectId = this.keyValue.get(this.keysToGet[0]); if (projectId == null || projectId.isEmpty()) { @@ -558,22 +335,6 @@ private String getUserId() { return "system"; } - private String determineTriggerType(String resumeRunId) { - if (resumeRunId != null && !resumeRunId.isEmpty()) { - return AutomationConstants.TRIGGER_RESUME; - } - // Explicit triggerType param (webhook, storage-poll, db-poll) takes precedence - String explicit = this.keyValue.get(this.keysToGet[3]); - if (explicit != null && !explicit.isBlank()) { - return explicit.toUpperCase().replace("-", "_"); - } - String manual = this.keyValue.get(this.keysToGet[1]); - if ("true".equalsIgnoreCase(manual)) { - return AutomationConstants.TRIGGER_MANUAL; - } - return AutomationConstants.TRIGGER_SCHEDULED; - } - private Map buildInitialScope(String runId) { Map scope = new HashMap<>(); String now = Instant.now().toString(); @@ -584,8 +345,13 @@ private Map buildInitialScope(String runId) { } private Timestamp toTimestamp(Instant instant) { - return Utility.getSqlTimestampUTC( - LocalDateTime.ofInstant(instant, ZoneOffset.UTC)); + return Utility.getSqlTimestampUTC(LocalDateTime.ofInstant(instant, ZoneOffset.UTC)); + } + + private static String generatePreview(String s) { + if (s == null) return null; + return s.length() <= AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH + ? s : s.substring(0, AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH); } // -- Result Building ----------------------------------------------------------- @@ -597,19 +363,14 @@ private Map buildNodeResult(String nodeId, String nodeLabel, result.put(AutomationConstants.NODE_LABEL, nodeLabel); result.put(AutomationConstants.STATUS, status); result.put(AutomationConstants.DURATION_MS, durationMs); - if (outputPreview != null) { - result.put(AutomationConstants.OUTPUT_PREVIEW, outputPreview); - } - if (errorMessage != null) { - result.put(AutomationConstants.ERROR_MESSAGE, errorMessage); - } + if (outputPreview != null) result.put(AutomationConstants.OUTPUT_PREVIEW, outputPreview); + if (errorMessage != null) result.put(AutomationConstants.ERROR_MESSAGE, errorMessage); return result; } private Map buildRunResult(String runId, String projectId, String status, int totalNodes, int completedNodes, String failedNodeId, List> nodeResults) { - // Read actual timestamps from DB rather than synthesizing "now" on every call. Map stored = AutomationDatabaseUtility.getRunDetail(runId); Map result = new HashMap<>(); result.put(AutomationConstants.RUN_ID, runId); @@ -621,9 +382,7 @@ private Map buildRunResult(String runId, String projectId, Strin result.put(AutomationConstants.STARTED_AT, stored.get(AutomationConstants.STARTED_AT)); result.put(AutomationConstants.COMPLETED_AT, stored.get(AutomationConstants.COMPLETED_AT)); } - if (failedNodeId != null) { - result.put(AutomationConstants.FAILED_NODE_ID, failedNodeId); - } + if (failedNodeId != null) result.put(AutomationConstants.FAILED_NODE_ID, failedNodeId); result.put("nodeResults", nodeResults); return result; } diff --git a/src/prerna/reactor/automation/nodes/AutomationNodeContext.java b/src/prerna/reactor/automation/nodes/AutomationNodeContext.java index f76d62ef44e..165e5d04bb7 100644 --- a/src/prerna/reactor/automation/nodes/AutomationNodeContext.java +++ b/src/prerna/reactor/automation/nodes/AutomationNodeContext.java @@ -27,49 +27,13 @@ *******************************************************************************/ package prerna.reactor.automation.nodes; -import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import prerna.om.Insight; /** - * Single param object bundling everything an {@link IAutomationNodeExecutor} needs to run one - * node - replacing the inconsistent, differently-shaped per-method argument lists the previous - * {@code executeXNode} private methods each had (e.g. {@code executeWaitNode(node, scope, - * configMap)} took 3 args, {@code executeSwitchNode(runId, node, scope, configMap, - * ancestorProjectIds)} took 5). - * - *

Immutable - {@code scope} and {@code ancestorProjectIds} are references to the caller's - * live, mutable collections (executors write node outputs back into {@code scope} exactly as - * the previous {@code executeXNode} methods did), but the context object itself carries no - * mutable state of its own. - * - * @param runId the current run's id - * @param projectId the project id this run belongs to - * @param node this node's full definition from {@code automation.json} - * ({@code id}, {@code label}, {@code type}, {@code config}, ...) - * @param scope the run's current execution scope (prior node outputs, keyed by - * {@code outputVar}) - mutable, shared with the caller - * @param configMap the automation's {@code automation-config.json} key/value pairs - * @param ancestorProjectIds the chain of project ids already executing on this call stack, - * including this run's own project id - used by - * {@code SubAutomationNodeExecutor} for the self/transitive-call cycle - * guard - * @param insight the execution context - engines/reactors this node calls need it - * @param cancelFlag the run's cancellation flag, already resolved once by the caller - * (cluster-safe: reflects both the local {@code AtomicBoolean} and - * the DB {@code CANCEL_REQUESTED} column at the time this node - * started) - executors that loop internally (e.g. a future retry/ - * backoff inside a single node) should check this between iterations - * @param nodeDispatcher recurses into a single inner/branch node - see {@link NodeDispatcher}. - * Only used by composite executors (conditional, while-loop, try-catch, - * switch, retry, parallel); {@code null} is never passed - executors - * that don't need it simply don't call it - * @param childAutomationRunner runs an entire target project's automation graph as a nested child - * run - see {@link ChildAutomationRunner}. Only used by - * {@code SubAutomationNodeExecutor} + * Single param object bundling everything an {@link IAutomationNodeExecutor} needs to run one node. */ public record AutomationNodeContext( String runId, @@ -77,49 +41,25 @@ public record AutomationNodeContext( Map node, Map scope, Map configMap, - Set ancestorProjectIds, Insight insight, - AtomicBoolean cancelFlag, - NodeDispatcher nodeDispatcher, - ChildAutomationRunner childAutomationRunner) { + AtomicBoolean cancelFlag) { - /** Convenience accessor for this node's {@code id} field. */ public String nodeId() { return (String) node.get("id"); } - /** Convenience accessor for this node's {@code label} field. */ public String nodeLabel() { Object label = node.get("label"); return label != null ? label.toString() : "unnamed"; } - /** Convenience accessor for this node's {@code type} field. */ public String nodeType() { return (String) node.get("type"); } - /** Convenience accessor for this node's {@code config} field, or an empty map if absent. */ @SuppressWarnings("unchecked") public Map config() { Object config = node.get("config"); return config instanceof Map ? (Map) config : Map.of(); } - - /** - * Same list-of-strings shape used throughout the previous {@code executeXNode} methods for - * a branch/loop/case's inner nodes (e.g. {@code trueGraph.nodes}, {@code subGraph.nodes}). - */ - @SuppressWarnings("unchecked") - public static List> graphNodes(Map graph) { - return graph != null && graph.get("nodes") instanceof List - ? (List>) graph.get("nodes") : null; - } - - /** See {@link #graphNodes(Map)} - the corresponding {@code edges} list. */ - @SuppressWarnings("unchecked") - public static List> graphEdges(Map graph) { - return graph != null && graph.get("edges") instanceof List - ? (List>) graph.get("edges") : null; - } } diff --git a/src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java b/src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java new file mode 100644 index 00000000000..961b93c8bbc --- /dev/null +++ b/src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java @@ -0,0 +1,50 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +import prerna.reactor.automation.AutomationConstants; + +/** + * Shared registry of stateless node executor instances. + * Used by both TriggerAutomationReactor and RunAutomationNodeReactor. + */ +public final class AutomationNodeExecutors { + + private AutomationNodeExecutors() {} + + public static final Map EXECUTORS = Map.of( + AutomationConstants.NODE_WAIT, new WaitNodeExecutor(), + AutomationConstants.NODE_DATABASE_ENGINE, new DatabaseEngineNodeExecutor(), + AutomationConstants.NODE_MODEL_ENGINE, new ModelEngineNodeExecutor(), + AutomationConstants.NODE_VECTOR_ENGINE, new VectorEngineNodeExecutor(), + AutomationConstants.NODE_STORAGE_ENGINE, new StorageEngineNodeExecutor(), + AutomationConstants.NODE_FUNCTION_ENGINE, new FunctionEngineNodeExecutor() + ); +} diff --git a/src/prerna/reactor/automation/nodes/ChildAutomationRunner.java b/src/prerna/reactor/automation/nodes/ChildAutomationRunner.java deleted file mode 100644 index 6ab78ebbe26..00000000000 --- a/src/prerna/reactor/automation/nodes/ChildAutomationRunner.java +++ /dev/null @@ -1,70 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.reactor.automation.nodes; - -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * Callback bound to {@code TriggerAutomationReactor.executeNodes}, used by - * {@code SubAutomationNodeExecutor} to run a target project's entire automation graph to completion - * as a nested, synchronous child run - distinct from {@link NodeDispatcher}, which runs a single - * inner node. - * - *

This is real orchestration (its own cancellation flag, its own heartbeat, its own DB run - * row/active-run claim) and intentionally stays owned by {@code TriggerAutomationReactor} rather - * than being reimplemented by the node-executor layer - see the "what does NOT change" scoping - * note on ticket #2746. - */ -@FunctionalInterface -public interface ChildAutomationRunner { - - /** - * Runs an already-claimed, already-topo-sorted child automation run to completion and returns - * its final run result (the same shape {@code buildRunResult} produces for a top-level run). - * - * @param childRunId the child run's id (already inserted into {@code AUTOMATION_RUNS} - * and already holding the active-run claim for {@code targetProjectId}) - * @param targetProjectId the project id whose automation is being run - * @param orderedNodes the child automation's nodes, already topologically sorted - * @param configMap the child project's {@code automation-config.json} key/value pairs - * @param priorOutputs prior node outputs to skip on resume - empty for a fresh - * sub-automation call, never itself resumable independently - * @param extraInitialScope the resolved {@code inputMapping} values to seed into the - * child's initial scope - * @param ancestorProjectIds the chain of project ids already executing on this call stack, - * including {@code targetProjectId} - used for the - * self/transitive-call cycle guard on any further nested calls - * @return the child run's final result map, containing at minimum {@code STATUS} - */ - Map run(String childRunId, String targetProjectId, - List> orderedNodes, Map configMap, - Map priorOutputs, Map extraInitialScope, - Set ancestorProjectIds); -} diff --git a/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java index 033f769d918..66ca23d4684 100644 --- a/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java @@ -27,46 +27,91 @@ *******************************************************************************/ package prerna.reactor.automation.nodes; -import prerna.reactor.automation.AutomationExecutionUtils; -import prerna.reactor.automation.PixelExecutionUtils; - +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; -/** - * Executes a "database-engine" node: builds and runs a {@code SqlQuery(...)} Pixel call from - * structured {@code config} on the backend, instead of trusting a frontend-precompiled - * {@code builtPixel} string (ticket #2743). Reuses {@code SqlQueryReactor}/ - * {@code AbstractSqlQueryReactor} unmodified via the normal Pixel path - including its existing - * SELECT-vs-mutation permission split ({@code userCanViewEngine} for reads, - * {@code userCanEditEngine} for writes) - so no security logic is duplicated here. - * - *

Config: {@code {engineId, operation: "read"|"write", expression (the SQL), limit, commit}}. - */ +import prerna.engine.api.IRDBMSEngine; +import prerna.reactor.automation.AutomationExecutionUtils; +import prerna.util.Utility; + public final class DatabaseEngineNodeExecutor implements IAutomationNodeExecutor { @Override - public Object execute(AutomationNodeContext ctx) { + public Object execute(AutomationNodeContext ctx) throws Exception { Map config = ctx.config(); String nodeLabel = ctx.nodeLabel(); - - String engineId = EngineNodeSupport.required(config, "engineId", "Database-engine", nodeLabel); - String sql = EngineNodeSupport.required(config, "expression", "Database-engine", nodeLabel); - String operation = EngineNodeSupport.optional(config, "operation", "read"); - Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String encodedEngineId = EngineNodeSupport.resolveEncoded(engineId, scope, configMap); - String encodedSql = EngineNodeSupport.resolveEncoded(sql, scope, configMap); - String pixel; + String engineId = required(config, "engineId", nodeLabel); + String sql = required(config, "expression", nodeLabel); + String operation = optional(config, "operation", "read"); + + String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); + String resolvedSql = AutomationExecutionUtils.resolve(sql, scope, configMap); + + IRDBMSEngine engine = (IRDBMSEngine) Utility.getEngine(resolvedEngineId); + if (engine == null) { + throw new IllegalArgumentException("Database-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); + } + if ("write".equals(operation)) { - pixel = "SqlQuery(database=[" + encodedEngineId + "], query=[" + encodedSql + "], commit=[true]);"; + try (Connection conn = engine.getConnection(); + PreparedStatement ps = conn.prepareStatement(resolvedSql)) { + int rowsAffected = ps.executeUpdate(); + return Map.of("rowsAffected", rowsAffected); + } catch (SQLException e) { + throw new IllegalStateException("Database-engine node \"" + nodeLabel + "\": write failed: " + e.getMessage(), e); + } } else { - int limit = EngineNodeSupport.optionalInt(config, "limit", 50); - pixel = "SqlQuery(database=[" + encodedEngineId + "], query=[" + encodedSql + "], limit=[" + limit + "]);"; + int limit = optionalInt(config, "limit", 50); + try (Connection conn = engine.getConnection(); + PreparedStatement ps = conn.prepareStatement(resolvedSql)) { + try (ResultSet rs = ps.executeQuery()) { + ResultSetMetaData meta = rs.getMetaData(); + int colCount = meta.getColumnCount(); + List> rows = new ArrayList<>(); + int count = 0; + while (rs.next() && count < limit) { + Map row = new LinkedHashMap<>(); + for (int i = 1; i <= colCount; i++) { + row.put(meta.getColumnLabel(i), rs.getObject(i)); + } + rows.add(row); + count++; + } + return rows; + } + } catch (SQLException e) { + throw new IllegalStateException("Database-engine node \"" + nodeLabel + "\": query failed: " + e.getMessage(), e); + } + } + } + + private static String required(Map config, String key, String nodeLabel) { + Object v = config.get(key); + if (v == null || v.toString().isBlank()) { + throw new IllegalArgumentException("Database-engine node \"" + nodeLabel + "\": '" + key + "' is required"); } + return v.toString(); + } + + private static String optional(Map config, String key, String def) { + Object v = config.get(key); + return (v == null || v.toString().isBlank()) ? def : v.toString(); + } - int timeoutSeconds = AutomationExecutionUtils.getNodeTimeout(ctx.node()); - return PixelExecutionUtils.runAndCollect(ctx.insight(), pixel, timeoutSeconds); + private static int optionalInt(Map config, String key, int def) { + Object v = config.get(key); + if (v == null) return def; + try { return Integer.parseInt(v.toString().trim()); } + catch (NumberFormatException e) { return def; } } } diff --git a/src/prerna/reactor/automation/nodes/EngineNodeSupport.java b/src/prerna/reactor/automation/nodes/EngineNodeSupport.java deleted file mode 100644 index 0704d5e0557..00000000000 --- a/src/prerna/reactor/automation/nodes/EngineNodeSupport.java +++ /dev/null @@ -1,172 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.reactor.automation.nodes; - -import java.util.Map; - -import com.google.gson.JsonElement; -import com.google.gson.JsonParser; -import com.google.gson.JsonSyntaxException; - -import prerna.reactor.automation.AutomationExecutionUtils; - -/** - * Shared helpers for the 5 engine-type node executors (database/model/vector/storage/function- - * engine). Each of those builds a validated Pixel call from structured {@code node.config} on - * the backend - reusing the target reactor's existing security/validation/query-building logic - * unmodified by running it through the normal Pixel path - rather than trusting a frontend- - * precompiled {@code builtPixel} string (see ticket #2743). - * - *

Every templated/dynamic value is wrapped in {@code ...}, not just the - * "obviously free text" fields (command/query/prompt) that the frontend's preview-only - * equivalent ({@code buildPixelPreview()} in {@code automation-utils.ts}) encodes. Any config field - * can carry a {@code ${var}} reference to an upstream node's output - which may itself be - * LLM-generated or attacker-influenced content - so any field that isn't a fixed structural - * literal (an engine id selected from a dropdown is still encoded defensively) gets the same - * injection protection. This is deliberately more conservative than the frontend preview - * builder - see epic sub-issue #2750 (verifying encode-wrapping coverage) for the general - * concern this addresses. - * - *

The exception is fields that must be embedded as a raw, unquoted Pixel map/list literal - * (e.g. {@code paramValues}, {@code metadata}) because the target reactor's key expects an - * actual parsed Map/List noun, not a String - {@code } would change how the parser - * types the literal. Those fields go through {@link #resolveAndValidateJsonLiteral} instead: - * {@code ${var}} substitution happens first (on the raw field, not the whole assembled pixel - * string), then the result is validated as syntactically complete, balanced JSON before it is - * spliced in unquoted. This is a real defense, not just documentation of the gap - a resolved - * value that doesn't parse as balanced JSON (e.g. one containing {@code "], SomeReactor(x=["}) - * is rejected outright rather than silently embedded, so it cannot break out of the literal's - * boundaries and inject arbitrary Pixel syntax. - */ -final class EngineNodeSupport { - - private EngineNodeSupport() { - // static utility - no instantiation - } - - /** Wraps a value as an {@code }-protected Pixel string literal, e.g. {@code "foo"}. */ - static String encoded(Object value) { - return "\"" + (value != null ? value : "") + "\""; - } - - /** - * Resolves {@code ${var}} references in a raw config value, then wraps the already-resolved - * result in {@code ...}. Every engine-type executor resolves each field - * individually this way and builds its Pixel call from already-resolved pieces - matching - * the established convention elsewhere in this package (e.g. {@code EmailNodeExecutor}) - - * rather than assembling a still-templated pixel string and resolving it as one final pass. - * Resolving per-field first, and never resolving the assembled string a second time, avoids - * a subtle double-substitution risk: if a second whole-string resolve pass ran after this - * value was already embedded, and the resolved content happened to itself contain a literal - * {@code ${...}} sequence matching a scope key (e.g. upstream data that isn't a template but - * looks like one), it would get incorrectly re-substituted. - */ - static String resolveEncoded(String rawTemplate, Map scope, Map configMap) { - return encoded(AutomationExecutionUtils.resolve(rawTemplate, scope, configMap)); - } - - /** - * Resolves {@code ${var}} references in a raw config value that will be spliced into a - * Pixel map/list literal position unquoted (e.g. {@code paramValues=[]}), - * then validates the resolved text is syntactically complete, balanced JSON (an object or - * array). Throws rather than returning unvalidated content, since a value that isn't - * well-formed, self-contained JSON could contain a sequence that breaks out of the literal - * and injects arbitrary Pixel syntax once embedded into the assembled pixel string. - */ - static String resolveAndValidateJsonLiteral(String rawTemplate, Map scope, - Map configMap, String fieldName, String nodeTypeLabel, String nodeLabel) { - String resolved = AutomationExecutionUtils.resolve(rawTemplate, scope, configMap); - try { - JsonElement el = JsonParser.parseString(resolved); - if (!el.isJsonObject() && !el.isJsonArray()) { - throw new IllegalArgumentException("must be a JSON object or array, got: " + resolved); - } - } catch (JsonSyntaxException | IllegalArgumentException e) { - throw new IllegalArgumentException(nodeTypeLabel + " node \"" + nodeLabel + "\": '" + fieldName + - "' did not resolve to valid, complete JSON after substituting ${var} references (" + - e.getMessage() + ") - refusing to embed unvalidated content into the Pixel call", e); - } - return resolved; - } - - /** - * Resolves {@code ${var}} references in a raw config value that will be embedded as a - * quoted, quote-escaped Pixel string (e.g. {@code map=[""]}, used by - * {@code FunctionEngineNodeExecutor}), then escapes the resolved text for that position. - * Same ordering rationale as {@link #resolveAndValidateJsonLiteral}: resolve the field alone - * first, escape the actual resolved content, then splice - not escape-then-resolve, which - * would let a substituted value's own quotes reach the pixel string unescaped and break out - * of the surrounding {@code "..."} boundary. - */ - static String resolveAndEscapeForQuotedPixelString(String rawTemplate, Map scope, - Map configMap) { - String resolved = AutomationExecutionUtils.resolve(rawTemplate, scope, configMap); - return resolved.replace("\\", "\\\\").replace("\"", "\\\""); - } - - /** - * Reads a required config field as a String, throwing a clear, node-labeled error if it's - * missing or blank - backend-side re-validation, since an automation.json can be edited - * directly (API call bypassing the FE, or a future FE bug) and must never be trusted implicitly. - */ - static String required(Map config, String key, String nodeTypeLabel, String nodeLabel) { - Object v = config.get(key); - if (v == null || v.toString().isBlank()) { - throw new IllegalArgumentException(nodeTypeLabel + " node \"" + nodeLabel + "\": '" + key + "' is required"); - } - return v.toString(); - } - - /** Reads an optional config field as a String, or {@code def} if missing/blank. */ - static String optional(Map config, String key, String def) { - Object v = config.get(key); - return (v == null || v.toString().isBlank()) ? def : v.toString(); - } - - /** Reads an optional config field as a String, or {@code null} if missing/blank. */ - static String optional(Map config, String key) { - return optional(config, key, null); - } - - /** Reads an optional config field as an int, or {@code def} if missing/blank/unparseable. */ - static int optionalInt(Map config, String key, int def) { - Object v = config.get(key); - if (v == null) return def; - try { - return Integer.parseInt(v.toString().trim()); - } catch (NumberFormatException e) { - return def; - } - } - - /** Ensures a Pixel statement string ends with a semicolon, as the parser requires. */ - static String terminated(String pixel) { - String trimmed = pixel.trim(); - return trimmed.endsWith(";") ? trimmed : trimmed + ";"; - } -} diff --git a/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java index 5fbaf29b4cd..500c5c11e3e 100644 --- a/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java @@ -29,45 +29,55 @@ import java.util.Map; +import prerna.engine.api.IFunctionEngine; import prerna.reactor.automation.AutomationExecutionUtils; -import prerna.reactor.automation.PixelExecutionUtils; +import prerna.util.Utility; -/** - * Executes a "function-engine" node: builds and runs an {@code ExecuteFunctionEngine(...)} / - * {@code ExecuteStreamingFunctionEngine(...)} Pixel call from structured {@code config} on the - * backend, instead of trusting a frontend-precompiled {@code builtPixel} string (ticket #2743). - * Reuses the existing {@code ExecuteFunctionEngineReactor}/{@code ExecuteStreamingFunctionEngineReactor} - * unmodified via the normal Pixel path. - * - *

Config: {@code {engineId, operation: "default"|"streaming", params (a JSON object string)}}. - * - *

{@code params} is passed as a quoted, quote-escaped string (matching the frontend's existing - * {@code buildPixelPreview()} shape exactly, rather than {@code }-wrapped like other - * fields here) because the target reactor's {@code map} key expects a Pixel Map noun, and - * {@code } would change how the parser types the literal. {@code ${var}} substitution - * happens on the raw field first (via - * {@link EngineNodeSupport#resolveAndEscapeForQuotedPixelString}), then the resolved text is - * quote-escaped before it is embedded - so a substituted value's own quotes can't break out of - * the surrounding {@code "..."} boundary. - */ public final class FunctionEngineNodeExecutor implements IAutomationNodeExecutor { @Override - public Object execute(AutomationNodeContext ctx) { + public Object execute(AutomationNodeContext ctx) throws Exception { Map config = ctx.config(); String nodeLabel = ctx.nodeLabel(); Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String engineId = EngineNodeSupport.required(config, "engineId", "Function-engine", nodeLabel); - String operation = EngineNodeSupport.optional(config, "operation"); - String params = EngineNodeSupport.optional(config, "params", "{}"); - String escapedParams = EngineNodeSupport.resolveAndEscapeForQuotedPixelString(params, scope, configMap); - String command = "streaming".equals(operation) ? "ExecuteStreamingFunctionEngine" : "ExecuteFunctionEngine"; - String pixel = command + "(engine=[" + EngineNodeSupport.resolveEncoded(engineId, scope, configMap) + - "], map=[\"" + escapedParams + "\"]);"; + String engineId = required(config, "engineId", nodeLabel); + String params = optional(config, "params", "{}"); + + String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); + String resolvedParams = AutomationExecutionUtils.resolve(params, scope, configMap); + + IFunctionEngine engine = Utility.getFunctionEngine(resolvedEngineId); + if (engine == null) { + throw new IllegalArgumentException("Function-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); + } + + Map paramMap = parseParams(resolvedParams, nodeLabel); + return engine.execute(paramMap); + } + + @SuppressWarnings("unchecked") + private static Map parseParams(String json, String nodeLabel) { + if (json == null || json.isBlank()) return Map.of(); + try { + Map parsed = AutomationExecutionUtils.GSON.fromJson(json, Map.class); + return parsed != null ? parsed : Map.of(); + } catch (Exception e) { + throw new IllegalArgumentException("Function-engine node \"" + nodeLabel + "\": params is not valid JSON: " + e.getMessage(), e); + } + } + + private static String required(Map config, String key, String nodeLabel) { + Object v = config.get(key); + if (v == null || v.toString().isBlank()) { + throw new IllegalArgumentException("Function-engine node \"" + nodeLabel + "\": '" + key + "' is required"); + } + return v.toString(); + } - int timeoutSeconds = AutomationExecutionUtils.getNodeTimeout(ctx.node()); - return PixelExecutionUtils.runAndCollect(ctx.insight(), pixel, timeoutSeconds); + private static String optional(Map config, String key, String def) { + Object v = config.get(key); + return (v == null || v.toString().isBlank()) ? def : v.toString(); } } diff --git a/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java index 471ef1e27b4..8df09d75d29 100644 --- a/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java @@ -27,86 +27,85 @@ *******************************************************************************/ package prerna.reactor.automation.nodes; +import java.util.Arrays; +import java.util.List; import java.util.Map; +import prerna.engine.api.IModelEngine; +import prerna.engine.impl.model.responses.AskModelEngineResponse; +import prerna.engine.impl.model.responses.EmbeddingsModelEngineResponse; import prerna.reactor.automation.AutomationExecutionUtils; -import prerna.reactor.automation.PixelExecutionUtils; +import prerna.util.Utility; -/** - * Executes a "model-engine" node: builds and runs the matching model-operation Pixel call - * ({@code LLM}/{@code Embeddings}/{@code Vision}/{@code NER}) from structured {@code config} on - * the backend, instead of trusting a frontend-precompiled {@code builtPixel} string (ticket - * #2743). Reuses the existing {@code LLMReactor}/{@code EmbeddingsReactor}/{@code VisionReactor}/ - * {@code NERReactor} unmodified via the normal Pixel path. - * - *

Config: {@code {engineId, operation: "llm"|"embeddings"|"vision"|"ner", command, context, - * paramValues, values, image, prompt, entities}}. - */ public final class ModelEngineNodeExecutor implements IAutomationNodeExecutor { @Override - public Object execute(AutomationNodeContext ctx) { + public Object execute(AutomationNodeContext ctx) throws Exception { Map config = ctx.config(); String nodeLabel = ctx.nodeLabel(); Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String engineId = EngineNodeSupport.required(config, "engineId", "Model-engine", nodeLabel); - String operation = EngineNodeSupport.optional(config, "operation", "llm"); - String encodedEngineId = EngineNodeSupport.resolveEncoded(engineId, scope, configMap); - String pixel; + String engineId = required(config, "engineId", nodeLabel); + String operation = optional(config, "operation", "llm"); + String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); + + IModelEngine engine = Utility.getModel(resolvedEngineId); + if (engine == null) { + throw new IllegalArgumentException("Model-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); + } + switch (operation) { case "embeddings": { - String values = EngineNodeSupport.required(config, "values", "Model-engine", nodeLabel); - pixel = "Embeddings(engine=[" + encodedEngineId + - "], values=[" + EngineNodeSupport.resolveEncoded(values, scope, configMap) + "]);"; - break; - } - case "vision": { - String command = EngineNodeSupport.required(config, "command", "Model-engine", nodeLabel); - String image = EngineNodeSupport.required(config, "image", "Model-engine", nodeLabel); - pixel = "Vision(engine=[" + encodedEngineId + - "], command=[" + EngineNodeSupport.resolveEncoded(command, scope, configMap) + - "], image=[" + EngineNodeSupport.resolveEncoded(image, scope, configMap) + "]);"; - break; - } - case "ner": { - String prompt = EngineNodeSupport.required(config, "prompt", "Model-engine", nodeLabel); - String entities = EngineNodeSupport.required(config, "entities", "Model-engine", nodeLabel); - pixel = "NER(engine=[" + encodedEngineId + - "], prompt=[" + EngineNodeSupport.resolveEncoded(prompt, scope, configMap) + - "], entities=[" + EngineNodeSupport.resolveEncoded(entities, scope, configMap) + "]);"; - break; + String values = required(config, "values", nodeLabel); + String resolvedValues = AutomationExecutionUtils.resolve(values, scope, configMap); + List valueList = Arrays.asList(resolvedValues.split(",")); + EmbeddingsModelEngineResponse response = engine.embeddings(valueList, ctx.insight(), null); + return response.getResponse(); } default: { - // llm - String command = EngineNodeSupport.required(config, "command", "Model-engine", nodeLabel); - StringBuilder pixelBuilder = new StringBuilder("LLM(engine=[") - .append(encodedEngineId) - .append("], command=[").append(EngineNodeSupport.resolveEncoded(command, scope, configMap)).append("]"); - String context = EngineNodeSupport.optional(config, "context"); - if (context != null) { - pixelBuilder.append(", context=[").append(EngineNodeSupport.resolveEncoded(context, scope, configMap)).append("]"); - } - String paramValues = EngineNodeSupport.optional(config, "paramValues"); - if (paramValues != null) { - // paramValues is a Pixel map literal (e.g. {"key":"value"}), not a string - - // ReactorKeysEnum.PARAM_VALUES_MAP expects an actual map, so this must stay - // unquoted/un-encoded to match the FE's existing wire contract (buildPixelPreview - // emits it the same way). resolveAndValidateJsonLiteral substitutes ${var} - // refs on this field alone and rejects anything that doesn't resolve to - // complete, balanced JSON, so a value can't break out of the map literal and - // inject arbitrary Pixel syntax. - String resolvedParamValues = EngineNodeSupport.resolveAndValidateJsonLiteral( - paramValues, scope, configMap, "paramValues", "Model-engine", nodeLabel); - pixelBuilder.append(", paramValues=[").append(resolvedParamValues).append("]"); - } - pixelBuilder.append(");"); - pixel = pixelBuilder.toString(); + // llm (and vision/ner as fallback — both use ask() with the primary command field) + String command = required(config, "command", nodeLabel); + String resolvedCommand = AutomationExecutionUtils.resolve(command, scope, configMap); + String context = optional(config, "context"); + String resolvedContext = (context != null) + ? AutomationExecutionUtils.resolve(context, scope, configMap) : null; + Map params = parseParams(config, scope, configMap, nodeLabel); + @SuppressWarnings("rawtypes") + AskModelEngineResponse response = engine.ask(resolvedCommand, resolvedContext, ctx.insight(), params); + return response.getResponse(); } } + } + + @SuppressWarnings("unchecked") + private static Map parseParams(Map config, + Map scope, Map configMap, String nodeLabel) { + String paramValues = optional(config, "paramValues"); + if (paramValues == null) return null; + String resolved = AutomationExecutionUtils.resolve(paramValues, scope, configMap); + try { + return AutomationExecutionUtils.GSON.fromJson(resolved, Map.class); + } catch (Exception e) { + throw new IllegalArgumentException("Model-engine node \"" + nodeLabel + "\": paramValues is not valid JSON: " + e.getMessage(), e); + } + } + + private static String required(Map config, String key, String nodeLabel) { + Object v = config.get(key); + if (v == null || v.toString().isBlank()) { + throw new IllegalArgumentException("Model-engine node \"" + nodeLabel + "\": '" + key + "' is required"); + } + return v.toString(); + } + + private static String optional(Map config, String key) { + Object v = config.get(key); + return (v == null || v.toString().isBlank()) ? null : v.toString(); + } - int timeoutSeconds = AutomationExecutionUtils.getNodeTimeout(ctx.node()); - return PixelExecutionUtils.runAndCollect(ctx.insight(), pixel, timeoutSeconds); + private static String optional(Map config, String key, String def) { + Object v = config.get(key); + return (v == null || v.toString().isBlank()) ? def : v.toString(); } } diff --git a/src/prerna/reactor/automation/nodes/NodeDispatcher.java b/src/prerna/reactor/automation/nodes/NodeDispatcher.java deleted file mode 100644 index cf9c7e1135c..00000000000 --- a/src/prerna/reactor/automation/nodes/NodeDispatcher.java +++ /dev/null @@ -1,60 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.reactor.automation.nodes; - -import java.util.Map; - -/** - * Callback bound to {@code TriggerAutomationReactor.executeSingleNode}, allowing composite node - * executors (e.g. {@code ConditionalNodeExecutor}, {@code WhileLoopNodeExecutor}, - * {@code TryCatchNodeExecutor}, {@code SwitchNodeExecutor}, {@code RetryNodeExecutor}, - * {@code ParallelNodeExecutor}) to recurse into a branch/loop/case's inner nodes without - * depending on {@code TriggerAutomationReactor} directly. - * - *

{@code node}/{@code scope} are the only two arguments that vary per recursive call - the - * rest of the calling node's context ({@code runId}, {@code configMap}, {@code ancestorProjectIds}) - * stays fixed across the recursion and is captured by the lambda this is bound to. - */ -@FunctionalInterface -public interface NodeDispatcher { - - /** - * Executes a single inner node exactly as {@code executeSingleNode} would for a top-level - * node - markNodeRunning, timing, output-transform, preview, checkpointing, and - * success/failure result building all happen inside this call, matching the contract - * every branch of the original {@code if/else} chain relied on implicitly. - * - * @param node the inner node definition (from a {@code trueGraph}/{@code falseGraph}/ - * {@code subGraph}/{@code tryGraph}/{@code catchGraph}/case branch) - * @param scope the current execution scope - inner nodes read prior outputs from it and - * (via the caller) may write their own output back into it - * @return the same node-result map shape {@code executeSingleNode} normally returns - - * contains at minimum {@code STATUS}, and on success {@code outputValue} - */ - Map dispatch(Map node, Map scope); -} diff --git a/src/prerna/reactor/automation/nodes/PixelNodeExecutor.java b/src/prerna/reactor/automation/nodes/PixelNodeExecutor.java deleted file mode 100644 index f7802903594..00000000000 --- a/src/prerna/reactor/automation/nodes/PixelNodeExecutor.java +++ /dev/null @@ -1,101 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.reactor.automation.nodes; - -import java.util.Map; - -import prerna.reactor.automation.AutomationConstants; -import prerna.reactor.automation.AutomationExecutionUtils; -import prerna.reactor.automation.PixelExecutionUtils; - -/** - * Default/fallback executor - runs a node's frontend-precompiled {@code builtPixel} verbatim - * (after {@code ${var}} substitution). This is the correct behavior for node types that are - * genuinely arbitrary or composed Pixel with no single backing engine: - *

    - *
  • {@code trigger} - returns the run's trigger timestamp, no Pixel execution
    • - *
  • {@code app} - runs an arbitrary multi-engine recipe scoped to a project - * (e.g. {@code IndexPubmedDocuments(database=[..], storage=[..], vector=[..], ...)})
    • - *
  • {@code custom-pixel} - arbitrary user-authored Pixel, optionally scoped to an app via - * a leading {@code LoadApp(...)} setup call
    • - *
- * - *

Unlike the previous {@code executeNodePixel}, this is not used for - * {@code database-engine}/{@code model-engine}/{@code vector-engine}/{@code storage-engine}/ - * {@code function-engine} nodes - those have their own dedicated executors that read structured - * {@code config} and call the matching engine/reactor directly, rather than trusting a - * frontend-precompiled Pixel string (see ticket #2743). - */ -public final class PixelNodeExecutor implements IAutomationNodeExecutor { - - @Override - @SuppressWarnings("unchecked") - public Object execute(AutomationNodeContext ctx) { - Map node = ctx.node(); - Map scope = ctx.scope(); - Map configMap = ctx.configMap(); - String type = ctx.nodeType(); - - if (AutomationConstants.NODE_TRIGGER.equals(type)) { - return scope.get("triggered_at"); - } - - String builtPixel = (String) node.get("builtPixel"); - if (builtPixel == null || builtPixel.isBlank() || builtPixel.startsWith("//")) { - throw new IllegalStateException("Node \"" + node.get("label") + - "\" has no compiled pixel - please Save the automation before running"); - } - - int timeoutSeconds = AutomationExecutionUtils.getNodeTimeout(node); - String resolvedPixel = AutomationExecutionUtils.resolve(builtPixel, scope, configMap); - - // For custom-pixel nodes with an appId, the builtPixel is "LoadApp(...); actualPixel". - // Run LoadApp as a fire-and-forget setup step so only the actual pixel's output - // is captured and stored as the node's result. - if (AutomationConstants.NODE_CUSTOM_PIXEL.equals(type)) { - Map config = (Map) node.get("config"); - Object appIdObj = config != null ? config.get("appId") : null; - if (appIdObj != null && !appIdObj.toString().isBlank()) { - int semicolon = resolvedPixel.indexOf(';'); - if (semicolon > 0) { - String setupPixel = resolvedPixel.substring(0, semicolon).trim(); - String actualPixel = resolvedPixel.substring(semicolon + 1).trim(); - if (!setupPixel.isBlank() && !actualPixel.isBlank()) { - // SEMOSS pixel parser requires a trailing semicolon on every statement - if (!setupPixel.endsWith(";")) setupPixel += ";"; - if (!actualPixel.endsWith(";")) actualPixel += ";"; - ctx.insight().runPixel(setupPixel); // set context, discard output - return PixelExecutionUtils.runAndCollect(ctx.insight(), actualPixel, timeoutSeconds); - } - } - } - } - - return PixelExecutionUtils.runAndCollect(ctx.insight(), resolvedPixel, timeoutSeconds); - } -} diff --git a/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java index 9a2b9658c29..cdb7a145b36 100644 --- a/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java @@ -27,84 +27,81 @@ *******************************************************************************/ package prerna.reactor.automation.nodes; +import java.util.Base64; +import java.util.List; import java.util.Map; +import prerna.engine.api.IStorageEngine; import prerna.reactor.automation.AutomationExecutionUtils; -import prerna.reactor.automation.PixelExecutionUtils; +import prerna.util.Utility; -/** - * Executes a "storage-engine" node: builds and runs the matching storage-operation Pixel call - * from structured {@code config} on the backend, instead of trusting a frontend-precompiled - * {@code builtPixel} string (ticket #2743). Reuses the existing - * {@code ListStoragePathReactor}/{@code PullFromStorageReactor}/{@code PushToStorageReactor}/ - * {@code DeleteFromStorageReactor}/{@code GetStorageFileAsBase64Reactor} unmodified via the - * normal Pixel path. - * - *

Config: {@code {engineId, operation: "list"|"download"|"upload"|"delete"|"read-base64", - * storagePath, filePath, metadata}}. - */ public final class StorageEngineNodeExecutor implements IAutomationNodeExecutor { @Override - public Object execute(AutomationNodeContext ctx) { + public Object execute(AutomationNodeContext ctx) throws Exception { Map config = ctx.config(); String nodeLabel = ctx.nodeLabel(); Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String engineId = EngineNodeSupport.required(config, "engineId", "Storage-engine", nodeLabel); - String operation = EngineNodeSupport.optional(config, "operation", "list"); - String encodedEngineId = EngineNodeSupport.resolveEncoded(engineId, scope, configMap); - String pixel; + String engineId = required(config, "engineId", nodeLabel); + String operation = optional(config, "operation", "list"); + String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); + + IStorageEngine engine = Utility.getStorage(resolvedEngineId); + if (engine == null) { + throw new IllegalArgumentException("Storage-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); + } + switch (operation) { case "download": { - String storagePath = EngineNodeSupport.required(config, "storagePath", "Storage-engine", nodeLabel); - String filePath = EngineNodeSupport.required(config, "filePath", "Storage-engine", nodeLabel); - pixel = "PullFromStorage(storage=[" + encodedEngineId + - "], storagePath=[" + EngineNodeSupport.resolveEncoded(storagePath, scope, configMap) + - "], filePath=[" + EngineNodeSupport.resolveEncoded(filePath, scope, configMap) + "]);"; - break; + String storagePath = required(config, "storagePath", nodeLabel); + String filePath = required(config, "filePath", nodeLabel); + String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); + String resolvedFile = AutomationExecutionUtils.resolve(filePath, scope, configMap); + engine.copyToLocal(resolvedStorage, resolvedFile); + return "Downloaded: " + resolvedStorage; } case "upload": { - String storagePath = EngineNodeSupport.required(config, "storagePath", "Storage-engine", nodeLabel); - String filePath = EngineNodeSupport.required(config, "filePath", "Storage-engine", nodeLabel); - StringBuilder pixelBuilder = new StringBuilder("PushToStorage(storage=[") - .append(encodedEngineId) - .append("], storagePath=[").append(EngineNodeSupport.resolveEncoded(storagePath, scope, configMap)) - .append("], filePath=[").append(EngineNodeSupport.resolveEncoded(filePath, scope, configMap)).append("]"); - String metadata = EngineNodeSupport.optional(config, "metadata"); - if (metadata != null) { - // Pixel map literal, not a string - see ModelEngineNodeExecutor's paramValues - // handling for the same resolve-then-validate treatment. - String resolvedMetadata = EngineNodeSupport.resolveAndValidateJsonLiteral( - metadata, scope, configMap, "metadata", "Storage-engine", nodeLabel); - pixelBuilder.append(", metadata=[").append(resolvedMetadata).append("]"); - } - pixelBuilder.append(");"); - pixel = pixelBuilder.toString(); - break; + String storagePath = required(config, "storagePath", nodeLabel); + String filePath = required(config, "filePath", nodeLabel); + String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); + String resolvedFile = AutomationExecutionUtils.resolve(filePath, scope, configMap); + engine.copyToStorage(resolvedFile, resolvedStorage, null); + return "Uploaded: " + resolvedFile; } case "delete": { - String storagePath = EngineNodeSupport.required(config, "storagePath", "Storage-engine", nodeLabel); - pixel = "DeleteFromStorage(storage=[" + encodedEngineId + - "], storagePath=[" + EngineNodeSupport.resolveEncoded(storagePath, scope, configMap) + "]);"; - break; + String storagePath = required(config, "storagePath", nodeLabel); + String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); + engine.deleteFromStorage(resolvedStorage); + return "Deleted: " + resolvedStorage; } case "read-base64": { - String storagePath = EngineNodeSupport.required(config, "storagePath", "Storage-engine", nodeLabel); - pixel = "GetStorageFileAsBase64(storage=[" + encodedEngineId + - "], storagePath=[" + EngineNodeSupport.resolveEncoded(storagePath, scope, configMap) + "]);"; - break; + String storagePath = required(config, "storagePath", nodeLabel); + String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); + byte[] bytes = engine.readBlobToMemory(resolvedStorage); + return Base64.getEncoder().encodeToString(bytes); } default: { // list - String storagePath = EngineNodeSupport.optional(config, "storagePath", "/"); - pixel = "ListStoragePath(storage=[" + encodedEngineId + - "], storagePath=[" + EngineNodeSupport.resolveEncoded(storagePath, scope, configMap) + "]);"; + String storagePath = optional(config, "storagePath", "/"); + String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); + List files = engine.list(resolvedStorage); + return files; } } + } + + private static String required(Map config, String key, String nodeLabel) { + Object v = config.get(key); + if (v == null || v.toString().isBlank()) { + throw new IllegalArgumentException("Storage-engine node \"" + nodeLabel + "\": '" + key + "' is required"); + } + return v.toString(); + } - int timeoutSeconds = AutomationExecutionUtils.getNodeTimeout(ctx.node()); - return PixelExecutionUtils.runAndCollect(ctx.insight(), pixel, timeoutSeconds); + private static String optional(Map config, String key, String def) { + Object v = config.get(key); + return (v == null || v.toString().isBlank()) ? def : v.toString(); } } diff --git a/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java index 7c4ac012b95..4006a28cd85 100644 --- a/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java @@ -27,94 +27,80 @@ *******************************************************************************/ package prerna.reactor.automation.nodes; +import java.util.Arrays; +import java.util.List; import java.util.Map; +import prerna.engine.api.IVectorDatabaseEngine; import prerna.reactor.automation.AutomationExecutionUtils; -import prerna.reactor.automation.PixelExecutionUtils; +import prerna.util.Utility; -/** - * Executes a "vector-engine" node: builds and runs the matching vector-operation Pixel call from - * structured {@code config} on the backend, instead of trusting a frontend-precompiled - * {@code builtPixel} string (ticket #2743). Reuses the existing - * {@code VectorDatabaseQueryReactor}/{@code VectorAttachFileToSourceReactor}/ - * {@code CreateEmbeddingsFromVectorCSVFileReactor}/{@code ListDocumentsInVectorDatabaseReactor}/ - * {@code RemoveDocumentFromVectorDatabaseReactor}/{@code VectorFileDownloadReactor} unmodified via - * the normal Pixel path - including {@code CreateEmbeddingsFromVectorCSVFileReactor}'s - * substantial (~385 line) CSV-parsing/chunking logic, which this deliberately does not - * re-implement. - * - *

Config: {@code {engineId, operation: "search"|"add-file"|"add-csv"|"list"|"delete"| - * "download", command, limit, filePath, source, space, filePaths, paramValues, fileNames}}. - */ public final class VectorEngineNodeExecutor implements IAutomationNodeExecutor { @Override - public Object execute(AutomationNodeContext ctx) { + public Object execute(AutomationNodeContext ctx) throws Exception { Map config = ctx.config(); String nodeLabel = ctx.nodeLabel(); Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String engineId = EngineNodeSupport.required(config, "engineId", "Vector-engine", nodeLabel); - String operation = EngineNodeSupport.optional(config, "operation", "search"); - String encodedEngineId = EngineNodeSupport.resolveEncoded(engineId, scope, configMap); - String pixel; + String engineId = required(config, "engineId", nodeLabel); + String operation = optional(config, "operation", "search"); + String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); + + IVectorDatabaseEngine engine = Utility.getVectorDatabase(resolvedEngineId); + if (engine == null) { + throw new IllegalArgumentException("Vector-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); + } + switch (operation) { - case "add-file": { - String filePath = EngineNodeSupport.required(config, "filePath", "Vector-engine", nodeLabel); - StringBuilder pixelBuilder = new StringBuilder("VectorAttachFileToSource(engine=[") - .append(encodedEngineId) - .append("], filePath=[").append(EngineNodeSupport.resolveEncoded(filePath, scope, configMap)).append("]"); - String source = EngineNodeSupport.optional(config, "source"); - if (source != null) pixelBuilder.append(", source=[").append(EngineNodeSupport.resolveEncoded(source, scope, configMap)).append("]"); - String space = EngineNodeSupport.optional(config, "space"); - if (space != null) pixelBuilder.append(", space=[").append(EngineNodeSupport.resolveEncoded(space, scope, configMap)).append("]"); - pixelBuilder.append(");"); - pixel = pixelBuilder.toString(); - break; - } + case "add-file": case "add-csv": { - String filePaths = EngineNodeSupport.required(config, "filePaths", "Vector-engine", nodeLabel); - StringBuilder pixelBuilder = new StringBuilder("CreateEmbeddingsFromVectorCSVFile(engine=[") - .append(encodedEngineId) - .append("], filePaths=[").append(EngineNodeSupport.resolveEncoded(filePaths, scope, configMap)).append("]"); - String paramValues = EngineNodeSupport.optional(config, "paramValues"); - if (paramValues != null) { - // Pixel map literal, not a string - see ModelEngineNodeExecutor's paramValues - // handling for the same resolve-then-validate treatment. - String resolvedParamValues = EngineNodeSupport.resolveAndValidateJsonLiteral( - paramValues, scope, configMap, "paramValues", "Vector-engine", nodeLabel); - pixelBuilder.append(", paramValues=[").append(resolvedParamValues).append("]"); - } - pixelBuilder.append(");"); - pixel = pixelBuilder.toString(); - break; + String filePaths = required(config, "filePath", nodeLabel); + String resolvedPaths = AutomationExecutionUtils.resolve(filePaths, scope, configMap); + List paths = Arrays.asList(resolvedPaths.split(",")); + engine.addDocument(paths, null); + return "Added " + paths.size() + " file(s)"; } - case "list": - pixel = "ListDocumentsInVectorDatabase(engine=[" + encodedEngineId + "]);"; - break; - case "delete": { - String fileNames = EngineNodeSupport.required(config, "fileNames", "Vector-engine", nodeLabel); - pixel = "RemoveDocumentFromVectorDatabase(engine=[" + encodedEngineId + - "], fileNames=[" + EngineNodeSupport.resolveEncoded(fileNames, scope, configMap) + "]);"; - break; + case "list": { + List> docs = engine.listDocuments(null); + return docs; } - case "download": { - String fileNames = EngineNodeSupport.required(config, "fileNames", "Vector-engine", nodeLabel); - pixel = "VectorFileDownload(engine=[" + encodedEngineId + - "], fileNames=[" + EngineNodeSupport.resolveEncoded(fileNames, scope, configMap) + "]);"; - break; + case "delete": { + String fileNames = required(config, "fileNames", nodeLabel); + String resolvedNames = AutomationExecutionUtils.resolve(fileNames, scope, configMap); + List names = Arrays.asList(resolvedNames.split(",")); + engine.removeDocument(names, null); + return "Deleted " + names.size() + " file(s)"; } default: { // search - String command = EngineNodeSupport.required(config, "command", "Vector-engine", nodeLabel); - int limit = EngineNodeSupport.optionalInt(config, "limit", 5); - pixel = "VectorDatabaseQuery(engine=[" + encodedEngineId + - "], command=[" + EngineNodeSupport.resolveEncoded(command, scope, configMap) + "], limit=[" + limit + "]);"; + String command = required(config, "command", nodeLabel); + String resolvedCommand = AutomationExecutionUtils.resolve(command, scope, configMap); + int limit = optionalInt(config, "limit", 5); + List> results = engine.nearestNeighbor(ctx.insight(), resolvedCommand, limit, null); + return results; } } + } + + private static String required(Map config, String key, String nodeLabel) { + Object v = config.get(key); + if (v == null || v.toString().isBlank()) { + throw new IllegalArgumentException("Vector-engine node \"" + nodeLabel + "\": '" + key + "' is required"); + } + return v.toString(); + } + + private static String optional(Map config, String key, String def) { + Object v = config.get(key); + return (v == null || v.toString().isBlank()) ? def : v.toString(); + } - int timeoutSeconds = AutomationExecutionUtils.getNodeTimeout(ctx.node()); - return PixelExecutionUtils.runAndCollect(ctx.insight(), pixel, timeoutSeconds); + private static int optionalInt(Map config, String key, int def) { + Object v = config.get(key); + if (v == null) return def; + try { return Integer.parseInt(v.toString().trim()); } + catch (NumberFormatException e) { return def; } } } diff --git a/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java index 636a068a2be..4d435cb0bad 100644 --- a/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java @@ -50,7 +50,7 @@ public final class WaitNodeExecutor implements IAutomationNodeExecutor { @Override @SuppressWarnings("unchecked") - public Object execute(AutomationNodeContext ctx) { + public Object execute(AutomationNodeContext ctx) throws Exception { Map node = ctx.node(); Map config = (Map) node.get("config"); String nodeLabel = ctx.nodeLabel(); diff --git a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java index 458d864d813..e498b953610 100644 --- a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java +++ b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java @@ -279,7 +279,7 @@ public void createColumnsAndTypes() { Pair.with(JOB_ID, VARCHAR_200), Pair.with(JOB_GROUP, VARCHAR_200))); - // AUTOMATION_RUNS, AUTOMATION_NODE_OUTPUTS, AUTOMATION_FOREACH_ROWS - table/column DDL + // AUTOMATION_RUNS, AUTOMATION_NODE_OUTPUTS - table/column DDL // (CREATE TABLE, primary keys, indexes, addColumnIfNotExists migrations) is owned by // AutomationDatabaseUtility.initialize() (called by SMSSWebWatcher), not this class - but // every column must still be declared here too, or SelectQueryStruct-based reads against @@ -295,7 +295,6 @@ public void createColumnsAndTypes() { Pair.with(AutomationConstants.AUTOMATION_ID, VARCHAR_255), Pair.with(AutomationConstants.STATUS, VARCHAR_200), Pair.with(AutomationConstants.TRIGGER_TYPE, VARCHAR_200), - Pair.with(AutomationConstants.RESUMED_FROM_RUN, VARCHAR_255), Pair.with(AutomationConstants.STARTED_AT, TIMESTAMP), Pair.with(AutomationConstants.COMPLETED_AT, TIMESTAMP), Pair.with(AutomationConstants.FAILED_NODE_ID, VARCHAR_255), @@ -304,8 +303,6 @@ public void createColumnsAndTypes() { Pair.with(AutomationConstants.TOTAL_NODES, INTEGER), Pair.with(AutomationConstants.COMPLETED_NODES, INTEGER), Pair.with(AutomationConstants.CREATED_BY, VARCHAR_255), - Pair.with(AutomationConstants.PARENT_RUN_ID, VARCHAR_255), - Pair.with(AutomationConstants.PARENT_NODE_ID, VARCHAR_255), Pair.with(AutomationConstants.CANCEL_REQUESTED, BOOLEAN))); addTable(AutomationConstants.TABLE_AUTOMATION_NODE_OUTPUTS, Arrays.asList( @@ -320,18 +317,6 @@ public void createColumnsAndTypes() { Pair.with(AutomationConstants.OUTPUT_VAR, VARCHAR_255), Pair.with(AutomationConstants.OUTPUT_VALUE, CLOB), Pair.with(AutomationConstants.OUTPUT_PREVIEW, VARCHAR_2000), - Pair.with(AutomationConstants.ROW_COUNT, INTEGER), - Pair.with(AutomationConstants.ERROR_MESSAGE, CLOB))); - - addTable(AutomationConstants.TABLE_AUTOMATION_FOREACH_ROWS, Arrays.asList( - Pair.with(AutomationConstants.RUN_ID, VARCHAR_255), - Pair.with(AutomationConstants.NODE_ID, VARCHAR_255), - Pair.with(AutomationConstants.ROW_INDEX, INTEGER), - Pair.with(AutomationConstants.ROW_KEY, VARCHAR_1000), - Pair.with(AutomationConstants.STATUS, VARCHAR_200), - Pair.with(AutomationConstants.STARTED_AT, TIMESTAMP), - Pair.with(AutomationConstants.COMPLETED_AT, TIMESTAMP), - Pair.with(AutomationConstants.DURATION_MS, BIGINT), Pair.with(AutomationConstants.ERROR_MESSAGE, CLOB))); // @formatter:on } From c728e2ba69b40206d2cc87b913d3ade714691595 Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Fri, 24 Jul 2026 10:39:19 -0400 Subject: [PATCH 04/25] fix: adding in app node --- .../automation/AutomationConstants.java | 6 + .../automation/AutomationExecutionUtils.java | 12 + .../automation/PixelExecutionUtils.java | 232 ++++++++++++++++++ .../nodes/AutomationNodeExecutors.java | 3 +- .../automation/nodes/PixelNodeExecutor.java | 64 +++++ 5 files changed, 316 insertions(+), 1 deletion(-) create mode 100644 src/prerna/reactor/automation/PixelExecutionUtils.java create mode 100644 src/prerna/reactor/automation/nodes/PixelNodeExecutor.java diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index 3416f1eb557..b293a3288c3 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -104,6 +104,12 @@ private AutomationConstants() {} public static final String NODE_MODEL_ENGINE = "model-engine"; public static final String NODE_FUNCTION_ENGINE = "function-engine"; public static final String NODE_WAIT = "wait"; + public static final String NODE_APP = "app"; + + // -- Pixel execution defaults ---------------------------------------------------- + + /** Default max execution time (seconds) for a node's Pixel call before it's timed out. */ + public static final int DEFAULT_TIMEOUT_SECONDS = 300; // -- Data type constants (for table creation) ---------------------------------- diff --git a/src/prerna/reactor/automation/AutomationExecutionUtils.java b/src/prerna/reactor/automation/AutomationExecutionUtils.java index f2f02ed290a..4a66f88d786 100644 --- a/src/prerna/reactor/automation/AutomationExecutionUtils.java +++ b/src/prerna/reactor/automation/AutomationExecutionUtils.java @@ -90,6 +90,18 @@ public static String resolve(String template, Map scope, Map node) { + Object timeout = node.get("timeoutSeconds"); + if (timeout instanceof Number) { + return ((Number) timeout).intValue(); + } + return AutomationConstants.DEFAULT_TIMEOUT_SECONDS; + } + /** * Loads {@code automation-config.json} for a project and returns key->value pairs. * Returns an empty map if the file does not exist or cannot be parsed. diff --git a/src/prerna/reactor/automation/PixelExecutionUtils.java b/src/prerna/reactor/automation/PixelExecutionUtils.java new file mode 100644 index 00000000000..fd4b7ca83cf --- /dev/null +++ b/src/prerna/reactor/automation/PixelExecutionUtils.java @@ -0,0 +1,232 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.om.Insight; +import prerna.om.ThreadStore; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.sablecc2.om.task.ITask; + +/** + * Shared utility for executing Pixel expressions and returning clean, serializable results. + * + *

Handles the common pitfalls of raw {@code insight.runPixel()} calls: + *

    + *
  • {@link ITask} materialization - query reactors return lazy cursors that must be collected
    • + *
  • Timeout enforcement - prevents hung queries from blocking pipelines indefinitely
    • + *
  • Error extraction - detects {@link prerna.sablecc2.om.PixelOperationType#ERROR} in results
    • + *
  • Null-safe return - always returns a usable value
    • + *
+ */ +public final class PixelExecutionUtils { + + private static final Logger classLogger = LogManager.getLogger(PixelExecutionUtils.class); + + private PixelExecutionUtils() {} + + /** + * Executes a pixel expression and returns a clean, serializable result. + * + * @param insight the execution context (carries user, varstore, etc.) + * @param pixel the fully resolved pixel string to execute + * @param timeoutSeconds max execution time; 0 or negative means no timeout + * @return the pixel result as a serializable object (Map, List, String, Number, etc.), or null + * @throws AutomationNodeTimeoutException if execution exceeds the timeout + * @throws AutomationPixelException if the pixel returns an error result + */ + public static Object runAndCollect(Insight insight, String pixel, int timeoutSeconds) { + if (pixel == null || pixel.isBlank()) { + return null; + } + + classLogger.debug("Executing pixel (timeout={}s): {}", timeoutSeconds, + pixel.length() > 200 ? pixel.substring(0, 200) + "..." : pixel); + + NounMetadata result; + if (timeoutSeconds > 0) { + result = executeWithTimeout(insight, pixel, timeoutSeconds); + } else { + result = executeDirectly(insight, pixel); + } + + if (result == null) { + return null; + } + + checkForError(result, pixel); + return materializeValue(result); + } + + /** Overload with default timeout from {@link AutomationConstants#DEFAULT_TIMEOUT_SECONDS}. */ + public static Object runAndCollect(Insight insight, String pixel) { + return runAndCollect(insight, pixel, AutomationConstants.DEFAULT_TIMEOUT_SECONDS); + } + + /** Serializes a pixel result to a JSON string for DB storage. */ + public static String serializeResult(Object result) { + if (result == null) return ""; + if (result instanceof String) return (String) result; + return AutomationExecutionUtils.GSON.toJson(result); + } + + /** + * Generates a truncated preview string for quick UI display. + * Returns null if input is null. + */ + public static String generatePreview(String serializedOutput) { + if (serializedOutput == null) return null; + int maxLength = AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH; + return serializedOutput.length() <= maxLength + ? serializedOutput + : serializedOutput.substring(0, maxLength); + } + + // -- Private implementation ---------------------------------------------------- + + private static NounMetadata executeWithTimeout(Insight insight, String pixel, int timeoutSeconds) { + // A new executor is created per timed call and shut down immediately after — no leak. + ExecutorService executor = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "automation-pixel-exec"); + t.setDaemon(true); + return t; + }); + + // ThreadStore is a plain ThreadLocal — not inherited by pool threads. + // Snapshot the caller's context so the worker thread has user/session/insight access. + Map callerContext = ThreadStore.getTheadMapObject(); + final Map contextSnapshot = + callerContext != null ? new HashMap<>(callerContext) : null; + + try { + Callable task = () -> { + if (contextSnapshot != null && !contextSnapshot.isEmpty()) { + ThreadStore.getInsightId(); + ThreadStore.setThreadMapObject(contextSnapshot); + } + try { + return executeDirectly(insight, pixel); + } finally { + ThreadStore.remove(); + } + }; + + Future future = executor.submit(task); + try { + return future.get(timeoutSeconds, TimeUnit.SECONDS); + } catch (TimeoutException e) { + future.cancel(true); + throw new AutomationNodeTimeoutException(pixel, timeoutSeconds); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) throw (RuntimeException) cause; + throw new IllegalStateException("Pixel execution failed: " + cause.getMessage(), cause); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Pixel execution interrupted", e); + } + } finally { + executor.shutdownNow(); + } + } + + private static NounMetadata executeDirectly(Insight insight, String pixel) { + List results = insight.runPixel(pixel).getResults(); + if (results == null || results.isEmpty()) return null; + // For multi-statement pixels the meaningful result is the last one. + return results.get(results.size() - 1); + } + + private static void checkForError(NounMetadata result, String pixel) { + if (result.getOpType() != null && result.getOpType().contains(PixelOperationType.ERROR)) { + String errorMsg = result.getValue() != null ? result.getValue().toString() : "Unknown pixel error"; + throw new AutomationPixelException(pixel, errorMsg); + } + } + + private static Object materializeValue(NounMetadata result) { + Object value = result.getValue(); + if (value == null) return null; + + if (value instanceof ITask) { + classLogger.debug("Materializing ITask result"); + try { + return ((ITask) value).collect(false); + } catch (Exception e) { + classLogger.error("Failed to materialize ITask: {}", e.getMessage(), e); + throw new IllegalStateException("Failed to materialize query result: " + e.getMessage(), e); + } + } + return value; + } + + // -- Exception types ----------------------------------------------------------- + + /** Thrown when a pixel execution exceeds its configured timeout. */ + public static class AutomationNodeTimeoutException extends RuntimeException { + private final int timeoutSeconds; + + public AutomationNodeTimeoutException(String pixel, int timeoutSeconds) { + super("Pixel execution timed out after " + timeoutSeconds + " seconds: " + + (pixel.length() > 100 ? pixel.substring(0, 100) + "..." : pixel)); + this.timeoutSeconds = timeoutSeconds; + } + + public int getTimeoutSeconds() { + return timeoutSeconds; + } + } + + /** Thrown when a pixel returns an ERROR operation type. */ + public static class AutomationPixelException extends RuntimeException { + private final String pixel; + + public AutomationPixelException(String pixel, String errorMessage) { + super(errorMessage); + this.pixel = pixel; + } + + public String getPixel() { + return pixel; + } + } +} diff --git a/src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java b/src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java index 961b93c8bbc..cc430cdc939 100644 --- a/src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java +++ b/src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java @@ -45,6 +45,7 @@ AutomationConstants.NODE_DATABASE_ENGINE, new DatabaseEngineNodeExecutor(), AutomationConstants.NODE_MODEL_ENGINE, new ModelEngineNodeExecutor(), AutomationConstants.NODE_VECTOR_ENGINE, new VectorEngineNodeExecutor(), AutomationConstants.NODE_STORAGE_ENGINE, new StorageEngineNodeExecutor(), - AutomationConstants.NODE_FUNCTION_ENGINE, new FunctionEngineNodeExecutor() + AutomationConstants.NODE_FUNCTION_ENGINE, new FunctionEngineNodeExecutor(), + AutomationConstants.NODE_APP, new PixelNodeExecutor() ); } diff --git a/src/prerna/reactor/automation/nodes/PixelNodeExecutor.java b/src/prerna/reactor/automation/nodes/PixelNodeExecutor.java new file mode 100644 index 00000000000..7248c570314 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/PixelNodeExecutor.java @@ -0,0 +1,64 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +import prerna.reactor.automation.AutomationConstants; +import prerna.reactor.automation.AutomationExecutionUtils; +import prerna.reactor.automation.PixelExecutionUtils; + +/** + * Executor for {@code app}-type nodes - runs a node's frontend-precompiled {@code builtPixel} + * verbatim (after {@code ${var}} substitution). This is the correct behavior for {@code app} + * nodes because they represent an arbitrary, multi-engine recipe scoped to a project (e.g. + * {@code LoadApp(project=[...]); IndexPubmedDocuments(database=[..], storage=[..], vector=[..], + * ...)}) with no single backing engine to dispatch to - unlike {@code database-engine}/ + * {@code model-engine}/{@code vector-engine}/{@code storage-engine}/{@code function-engine} + * nodes, which read structured {@code config} and call the matching engine/reactor directly. + */ +public final class PixelNodeExecutor implements IAutomationNodeExecutor { + + @Override + public Object execute(AutomationNodeContext ctx) { + Map node = ctx.node(); + Map scope = ctx.scope(); + Map configMap = ctx.configMap(); + + String builtPixel = (String) node.get("builtPixel"); + if (builtPixel == null || builtPixel.isBlank() || builtPixel.startsWith("//")) { + throw new IllegalStateException("Node \"" + node.get("label") + + "\" has no compiled pixel - please Save the automation before running"); + } + + int timeoutSeconds = AutomationExecutionUtils.getNodeTimeout(node); + String resolvedPixel = AutomationExecutionUtils.resolve(builtPixel, scope, configMap); + + return PixelExecutionUtils.runAndCollect(ctx.insight(), resolvedPixel, timeoutSeconds); + } +} From 0feebe87b4a61140f95c008a233778010cc2a320 Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Fri, 24 Jul 2026 12:29:46 -0400 Subject: [PATCH 05/25] fix: wire app node executor to config.pixel --- .../nodes/AppEngineNodeExecutor.java | 98 +++++++++++++++++++ .../nodes/AutomationNodeExecutors.java | 2 +- .../automation/nodes/PixelNodeExecutor.java | 64 ------------ 3 files changed, 99 insertions(+), 65 deletions(-) create mode 100644 src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java delete mode 100644 src/prerna/reactor/automation/nodes/PixelNodeExecutor.java diff --git a/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java new file mode 100644 index 00000000000..160717aba73 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java @@ -0,0 +1,98 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +import prerna.om.ThreadStore; +import prerna.project.api.IProject; +import prerna.reactor.automation.AutomationExecutionUtils; +import prerna.reactor.automation.PixelExecutionUtils; +import prerna.util.Utility; + +/** + * Executor for {@code app}-type nodes. Runs an arbitrary pixel expression, optionally + * inside a specific app's project context. + * + *

Config fields (from {@code node.config}): + *

    + *
  • {@code pixel} (required) — the pixel expression to run; supports {@code ${var}} substitution
    • + *
  • {@code appId} (optional) — if set, the pixel runs inside this project's insight context
    • + *
+ * + *

When {@code appId} is provided the project context overrides are set on {@link ThreadStore} + * before execution. {@link PixelExecutionUtils#runAndCollect} snapshots the caller's thread + * context (including these overrides) before submitting to the timeout thread, so the context + * propagates correctly even under timeout enforcement. + */ +public final class AppEngineNodeExecutor implements IAutomationNodeExecutor { + + @Override + public Object execute(AutomationNodeContext ctx) throws Exception { + Map config = ctx.config(); + String nodeLabel = ctx.nodeLabel(); + Map scope = ctx.scope(); + Map configMap = ctx.configMap(); + + String pixel = required(config, "pixel", nodeLabel); + String appId = optional(config, "appId"); + String resolvedPixel = AutomationExecutionUtils.resolve(pixel, scope, configMap); + String resolvedAppId = appId != null ? AutomationExecutionUtils.resolve(appId, scope, configMap) : null; + + if (resolvedAppId != null && !resolvedAppId.isBlank()) { + IProject project = Utility.getProject(resolvedAppId); + if (project == null) { + throw new IllegalArgumentException( + "App node \"" + nodeLabel + "\": project not found: " + resolvedAppId); + } + ThreadStore.setContextProjectIdOverride(resolvedAppId); + ThreadStore.setContextProjectNameOverride(project.getProjectName()); + try { + return PixelExecutionUtils.runAndCollect(ctx.insight(), resolvedPixel); + } finally { + ThreadStore.clearContextProjectOverride(); + } + } + + return PixelExecutionUtils.runAndCollect(ctx.insight(), resolvedPixel); + } + + private static String required(Map config, String key, String nodeLabel) { + Object v = config.get(key); + if (v == null || v.toString().isBlank()) { + throw new IllegalArgumentException( + "App node \"" + nodeLabel + "\": '" + key + "' is required"); + } + return v.toString(); + } + + private static String optional(Map config, String key) { + Object v = config.get(key); + return (v == null || v.toString().isBlank()) ? null : v.toString(); + } +} diff --git a/src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java b/src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java index cc430cdc939..52160781277 100644 --- a/src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java +++ b/src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java @@ -46,6 +46,6 @@ AutomationConstants.NODE_MODEL_ENGINE, new ModelEngineNodeExecutor(), AutomationConstants.NODE_VECTOR_ENGINE, new VectorEngineNodeExecutor(), AutomationConstants.NODE_STORAGE_ENGINE, new StorageEngineNodeExecutor(), AutomationConstants.NODE_FUNCTION_ENGINE, new FunctionEngineNodeExecutor(), - AutomationConstants.NODE_APP, new PixelNodeExecutor() + AutomationConstants.NODE_APP, new AppEngineNodeExecutor() ); } diff --git a/src/prerna/reactor/automation/nodes/PixelNodeExecutor.java b/src/prerna/reactor/automation/nodes/PixelNodeExecutor.java deleted file mode 100644 index 7248c570314..00000000000 --- a/src/prerna/reactor/automation/nodes/PixelNodeExecutor.java +++ /dev/null @@ -1,64 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.reactor.automation.nodes; - -import java.util.Map; - -import prerna.reactor.automation.AutomationConstants; -import prerna.reactor.automation.AutomationExecutionUtils; -import prerna.reactor.automation.PixelExecutionUtils; - -/** - * Executor for {@code app}-type nodes - runs a node's frontend-precompiled {@code builtPixel} - * verbatim (after {@code ${var}} substitution). This is the correct behavior for {@code app} - * nodes because they represent an arbitrary, multi-engine recipe scoped to a project (e.g. - * {@code LoadApp(project=[...]); IndexPubmedDocuments(database=[..], storage=[..], vector=[..], - * ...)}) with no single backing engine to dispatch to - unlike {@code database-engine}/ - * {@code model-engine}/{@code vector-engine}/{@code storage-engine}/{@code function-engine} - * nodes, which read structured {@code config} and call the matching engine/reactor directly. - */ -public final class PixelNodeExecutor implements IAutomationNodeExecutor { - - @Override - public Object execute(AutomationNodeContext ctx) { - Map node = ctx.node(); - Map scope = ctx.scope(); - Map configMap = ctx.configMap(); - - String builtPixel = (String) node.get("builtPixel"); - if (builtPixel == null || builtPixel.isBlank() || builtPixel.startsWith("//")) { - throw new IllegalStateException("Node \"" + node.get("label") + - "\" has no compiled pixel - please Save the automation before running"); - } - - int timeoutSeconds = AutomationExecutionUtils.getNodeTimeout(node); - String resolvedPixel = AutomationExecutionUtils.resolve(builtPixel, scope, configMap); - - return PixelExecutionUtils.runAndCollect(ctx.insight(), resolvedPixel, timeoutSeconds); - } -} From a51136f823c8ff21d6cf7a57418ea7a511afc63b Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Fri, 24 Jul 2026 14:17:35 -0400 Subject: [PATCH 06/25] chore: cleanup --- .../automation/AutomationConstants.java | 3 -- .../automation/AutomationDatabaseUtility.java | 44 ++----------------- .../automation/AutomationExecutionUtils.java | 13 +++--- .../GetAutomationConfigReactor.java | 1 - .../automation/GetAutomationRunReactor.java | 13 +++--- .../automation/ListAutomationRunsReactor.java | 5 --- .../automation/PixelExecutionUtils.java | 20 --------- .../automation/RunAutomationNodeReactor.java | 5 ++- .../SaveAutomationConfigReactor.java | 2 +- .../nodes/IAutomationNodeExecutor.java | 2 +- .../automation/nodes/WaitNodeExecutor.java | 4 +- .../scheduler/SchedulerOwlCreator.java | 11 ----- 12 files changed, 21 insertions(+), 102 deletions(-) diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index b293a3288c3..978302f6fd0 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -108,15 +108,12 @@ private AutomationConstants() {} // -- Pixel execution defaults ---------------------------------------------------- - /** Default max execution time (seconds) for a node's Pixel call before it's timed out. */ public static final int DEFAULT_TIMEOUT_SECONDS = 300; // -- Data type constants (for table creation) ---------------------------------- public static final String VARCHAR_255 = "VARCHAR(255)"; public static final String VARCHAR_500 = "VARCHAR(500)"; - public static final String VARCHAR_1000 = "VARCHAR(1000)"; - public static final String VARCHAR_2000 = "VARCHAR(2000)"; public static final String INTEGER = "INTEGER"; public static final String BIGINT = "BIGINT"; public static final String NOT_NULL = "NOT NULL"; diff --git a/src/prerna/reactor/automation/AutomationDatabaseUtility.java b/src/prerna/reactor/automation/AutomationDatabaseUtility.java index 3d6ce3c6440..f24e7a919a4 100644 --- a/src/prerna/reactor/automation/AutomationDatabaseUtility.java +++ b/src/prerna/reactor/automation/AutomationDatabaseUtility.java @@ -176,7 +176,6 @@ public static void markStaleRunsInterrupted() { IRDBMSEngine schedulerDb = getSchedulerDb(); if (schedulerDb == null) return; - // Use SelectQueryStruct to find stale runs SelectQueryStruct qs = new SelectQueryStruct(); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__RUN_ID", "RUN_ID")); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__PROJECT_ID", "PROJECT_ID")); @@ -189,7 +188,6 @@ public static void markStaleRunsInterrupted() { return; } - // For each running run, check if heartbeat is stale and mark as interrupted Timestamp threshold = toTimestamp(Instant.now().minusSeconds( AutomationConstants.STALE_HEARTBEAT_THRESHOLD_MINUTES * 60L)); Timestamp now = toTimestamp(Instant.now()); @@ -503,7 +501,7 @@ public static boolean updateHeartbeat(String runId, int completedNodes) { /** * Updates only the heartbeat timestamp for a running automation. - * Used during long-running for-each batches where completed node count hasn't changed. + * Used when the node count hasn't changed but liveness needs to be signaled. */ public static boolean touchHeartbeat(String runId) { IRDBMSEngine schedulerDb = getSchedulerDb(); @@ -597,38 +595,6 @@ public static Map getRunDetail(String runId) { // -- AUTOMATION_NODE_OUTPUTS CRUD ------------------------------------------------ - /** - * Inserts a node output record with PENDING status (before execution). - */ - public static boolean insertNodeOutput(String runId, String nodeId, String nodeLabel, int executionOrder) { - IRDBMSEngine schedulerDb = getSchedulerDb(); - if (schedulerDb == null) return false; - - Connection conn = null; - try { - conn = schedulerDb.getConnection(); - try (PreparedStatement ps = conn.prepareStatement(INSERT_NODE_OUTPUT)) { - int index = 1; - ps.setString(index++, runId); - ps.setString(index++, nodeId); - ps.setString(index++, nodeLabel); - ps.setInt(index++, executionOrder); - ps.setString(index++, AutomationConstants.NODE_STATUS_PENDING); - ps.executeUpdate(); - } - if (!conn.getAutoCommit()) { - conn.commit(); - } - return true; - } catch (SQLException e) { - classLogger.error("Failed to insert node output for run '{}', node '{}': {}", - runId, nodeId, e.getMessage(), e); - return false; - } finally { - closeConnection(schedulerDb, conn); - } - } - /** * Batch-inserts all node outputs for a run (all PENDING). */ @@ -771,9 +737,7 @@ public static boolean updateNodeFailed(String runId, String nodeId, Timestamp st } /** - * Gets all node outputs for a run (for scope reconstruction during resume). - * - * @return list of node output maps ordered by execution order + * Gets all node outputs for a run, ordered by execution order. */ public static List> getNodeOutputsForRun(String runId) { IRDBMSEngine schedulerDb = getSchedulerDb(); @@ -1000,9 +964,7 @@ private static void addPrimaryKeyIfNotExists(Connection conn, AbstractSqlQueryUt /** * Adds a column to an existing table if it isn't already present - used to migrate - * AUTOMATION_RUNS for installs that created the table before PARENT_RUN_ID/PARENT_NODE_ID - * existed. Safe to call unconditionally on every startup; errors (column already exists) - * are swallowed just like {@link #addPrimaryKeyIfNotExists}. + * installs that predate a column addition. Errors (column already exists) are swallowed. */ private static void addColumnIfNotExists(Connection conn, AbstractSqlQueryUtil queryUtil, String tableName, String columnName, String columnType) { diff --git a/src/prerna/reactor/automation/AutomationExecutionUtils.java b/src/prerna/reactor/automation/AutomationExecutionUtils.java index 4a66f88d786..65133be395d 100644 --- a/src/prerna/reactor/automation/AutomationExecutionUtils.java +++ b/src/prerna/reactor/automation/AutomationExecutionUtils.java @@ -50,9 +50,8 @@ /** * Shared static utilities for the automation execution engine. * - *

Centralizes logic shared across {@link TriggerAutomationReactor}, - * {@link RunAutomationNodeReactor}, and - * {@link prerna.reactor.automation.foreach.ForEachNodeExecutor}. + *

Centralizes logic shared across {@link TriggerAutomationReactor} and + * {@link RunAutomationNodeReactor}. */ public final class AutomationExecutionUtils { @@ -70,11 +69,6 @@ private AutomationExecutionUtils() {} /** * Resolves {@code ${varName}} and {@code ${config.KEY}} placeholders in a template string * via plain {@link String#replace} — no validation or escaping is applied. - * - *

Any substitution slot whose value can carry user-supplied or LLM-generated text MUST - * be wrapped in {@code ...} in the Pixel template before this is called — - * {@code PixelPreProcessor} handles decoding after parsing, preventing injected content from - * breaking the surrounding Pixel grammar. */ public static String resolve(String template, Map scope, Map configMap) { if (template == null) return ""; @@ -337,6 +331,9 @@ public static List> topoSort(List> nodes if (deg == 0) queue.add(neighbor); } } + if (sorted.size() != nodes.size()) { + throw new IllegalArgumentException("Automation graph contains a cycle — cannot determine execution order"); + } return sorted; } } diff --git a/src/prerna/reactor/automation/GetAutomationConfigReactor.java b/src/prerna/reactor/automation/GetAutomationConfigReactor.java index f931d02eaad..0eb1444f0d2 100644 --- a/src/prerna/reactor/automation/GetAutomationConfigReactor.java +++ b/src/prerna/reactor/automation/GetAutomationConfigReactor.java @@ -81,7 +81,6 @@ public NounMetadata execute() { try { String json = Files.readString(configFile.toPath(), StandardCharsets.UTF_8); - // strip sensitive values before returning List> entries = GSON.fromJson(json, new TypeToken>>() {}.getType()); if (entries != null) { for (Map entry : entries) { diff --git a/src/prerna/reactor/automation/GetAutomationRunReactor.java b/src/prerna/reactor/automation/GetAutomationRunReactor.java index 769b7252320..4d66cbeeb68 100644 --- a/src/prerna/reactor/automation/GetAutomationRunReactor.java +++ b/src/prerna/reactor/automation/GetAutomationRunReactor.java @@ -32,9 +32,6 @@ import java.util.List; import java.util.Map; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import prerna.auth.utils.SecurityProjectUtils; import prerna.reactor.AbstractReactor; import prerna.sablecc2.om.PixelDataType; @@ -47,12 +44,9 @@ *

Pixel: {@code GetAutomationRun(app=["appId"], runId=["uuid"])} * *

Reads from AUTOMATION_RUNS and AUTOMATION_NODE_OUTPUTS in the scheduler DB. - * Includes for-each progress for batch nodes. */ public class GetAutomationRunReactor extends AbstractReactor { - private static final Logger classLogger = LogManager.getLogger(GetAutomationRunReactor.class); - public GetAutomationRunReactor() { this.keysToGet = new String[]{ "project", "runId" }; this.keyRequired = new int[]{ 1, 1 }; @@ -64,6 +58,13 @@ public NounMetadata execute() { String projectId = this.keyValue.get(this.keysToGet[0]); String runId = this.keyValue.get(this.keysToGet[1]); + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + if (runId == null || runId.isEmpty()) { + throw new IllegalArgumentException("Must provide a run id"); + } + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { throw new IllegalArgumentException("Project does not exist or user does not have access"); diff --git a/src/prerna/reactor/automation/ListAutomationRunsReactor.java b/src/prerna/reactor/automation/ListAutomationRunsReactor.java index f707829f1da..b53f8c0efb8 100644 --- a/src/prerna/reactor/automation/ListAutomationRunsReactor.java +++ b/src/prerna/reactor/automation/ListAutomationRunsReactor.java @@ -31,9 +31,6 @@ import java.util.List; import java.util.Map; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import prerna.auth.utils.SecurityProjectUtils; import prerna.reactor.AbstractReactor; import prerna.sablecc2.om.PixelDataType; @@ -49,8 +46,6 @@ */ public class ListAutomationRunsReactor extends AbstractReactor { - private static final Logger classLogger = LogManager.getLogger(ListAutomationRunsReactor.class); - public ListAutomationRunsReactor() { this.keysToGet = new String[]{ "project", "limit" }; this.keyRequired = new int[]{ 1, 0 }; diff --git a/src/prerna/reactor/automation/PixelExecutionUtils.java b/src/prerna/reactor/automation/PixelExecutionUtils.java index fd4b7ca83cf..f8869b828e7 100644 --- a/src/prerna/reactor/automation/PixelExecutionUtils.java +++ b/src/prerna/reactor/automation/PixelExecutionUtils.java @@ -102,25 +102,6 @@ public static Object runAndCollect(Insight insight, String pixel) { return runAndCollect(insight, pixel, AutomationConstants.DEFAULT_TIMEOUT_SECONDS); } - /** Serializes a pixel result to a JSON string for DB storage. */ - public static String serializeResult(Object result) { - if (result == null) return ""; - if (result instanceof String) return (String) result; - return AutomationExecutionUtils.GSON.toJson(result); - } - - /** - * Generates a truncated preview string for quick UI display. - * Returns null if input is null. - */ - public static String generatePreview(String serializedOutput) { - if (serializedOutput == null) return null; - int maxLength = AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH; - return serializedOutput.length() <= maxLength - ? serializedOutput - : serializedOutput.substring(0, maxLength); - } - // -- Private implementation ---------------------------------------------------- private static NounMetadata executeWithTimeout(Insight insight, String pixel, int timeoutSeconds) { @@ -188,7 +169,6 @@ private static Object materializeValue(NounMetadata result) { if (value == null) return null; if (value instanceof ITask) { - classLogger.debug("Materializing ITask result"); try { return ((ITask) value).collect(false); } catch (Exception e) { diff --git a/src/prerna/reactor/automation/RunAutomationNodeReactor.java b/src/prerna/reactor/automation/RunAutomationNodeReactor.java index fa15906aa13..8e925d78928 100644 --- a/src/prerna/reactor/automation/RunAutomationNodeReactor.java +++ b/src/prerna/reactor/automation/RunAutomationNodeReactor.java @@ -168,8 +168,9 @@ private Map findNode(String projectId, String nodeId) { private Map buildScope(String contextRunId) { Map scope = new HashMap<>(); - scope.put("date", Instant.now().toString().substring(0, 10)); - scope.put("triggered_at", Instant.now().toString()); + String now = Instant.now().toString(); + scope.put("date", now.substring(0, 10)); + scope.put("triggered_at", now); if (contextRunId != null && !contextRunId.isEmpty()) { List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(contextRunId); diff --git a/src/prerna/reactor/automation/SaveAutomationConfigReactor.java b/src/prerna/reactor/automation/SaveAutomationConfigReactor.java index e1e10d92c27..d66997fde41 100644 --- a/src/prerna/reactor/automation/SaveAutomationConfigReactor.java +++ b/src/prerna/reactor/automation/SaveAutomationConfigReactor.java @@ -84,7 +84,7 @@ public NounMetadata execute() { String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); File configFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_CONFIG_FILE_NAME); - // GetAutomationConfig masks sensitive values (e.g. WEBHOOK_SECRET) as SENSITIVE_MASK. + // GetAutomationConfig masks sensitive values as SENSITIVE_MASK. // Restore the real values from disk so a config round-trip cannot silently destroy them. config = restoreMaskedSensitiveValues(config, configFile); diff --git a/src/prerna/reactor/automation/nodes/IAutomationNodeExecutor.java b/src/prerna/reactor/automation/nodes/IAutomationNodeExecutor.java index 8ee9645b86d..77ae0a1b50e 100644 --- a/src/prerna/reactor/automation/nodes/IAutomationNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/IAutomationNodeExecutor.java @@ -31,7 +31,7 @@ * Executes one automation node's operation, given all the context that node type needs. * *

One concrete implementation per {@code node.type} value (e.g. {@code WaitNodeExecutor}, - * {@code ConditionalNodeExecutor}, {@code DatabaseEngineNodeExecutor}), resolved via a + * {@code DatabaseEngineNodeExecutor}), resolved via a * {@code Map} registry in * {@link prerna.reactor.automation.TriggerAutomationReactor#executeSingleNode} instead of the * previous {@code if/else} chain keyed on {@code type}. diff --git a/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java index 4d435cb0bad..be309caddd8 100644 --- a/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java @@ -49,10 +49,8 @@ public final class WaitNodeExecutor implements IAutomationNodeExecutor { private static final int CANCEL_CHECK_INTERVAL_SECONDS = 5; @Override - @SuppressWarnings("unchecked") public Object execute(AutomationNodeContext ctx) throws Exception { - Map node = ctx.node(); - Map config = (Map) node.get("config"); + Map config = ctx.config(); String nodeLabel = ctx.nodeLabel(); String secondsTemplate = config.get("seconds") != null diff --git a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java index e498b953610..44560410023 100644 --- a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java +++ b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java @@ -124,7 +124,6 @@ import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_8; import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_80; import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_95; -import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_1000; import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_2000; import java.util.ArrayList; @@ -279,16 +278,6 @@ public void createColumnsAndTypes() { Pair.with(JOB_ID, VARCHAR_200), Pair.with(JOB_GROUP, VARCHAR_200))); - // AUTOMATION_RUNS, AUTOMATION_NODE_OUTPUTS - table/column DDL - // (CREATE TABLE, primary keys, indexes, addColumnIfNotExists migrations) is owned by - // AutomationDatabaseUtility.initialize() (called by SMSSWebWatcher), not this class - but - // every column must still be declared here too, or SelectQueryStruct-based reads against - // these tables fail with a NullPointerException resolving the conceptual->physical column - // name (AbstractSqlQueryUtil.isSelectorKeyword gets a null "selector" argument), since this - // OWL creator is this engine's only source of table/column metadata - there is no live - // schema-introspection fallback. Keep this column list in sync with AutomationDatabaseUtility's - // CREATE TABLE statements; needsRemake()/remakeOwl() automatically pick up any column added - // here on the next server startup, no manual OWL file deletion required. addTable(AutomationConstants.TABLE_AUTOMATION_RUNS, Arrays.asList( Pair.with(AutomationConstants.RUN_ID, VARCHAR_255), Pair.with(AutomationConstants.PROJECT_ID, VARCHAR_255), From 07cefe6f5dd6ad2c1a5b35f245fa460fa321d64d Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Fri, 24 Jul 2026 16:51:37 -0400 Subject: [PATCH 07/25] fix: output transformation --- .../automation/AutomationExecutionUtils.java | 60 ++++++++++++++++--- 1 file changed, 52 insertions(+), 8 deletions(-) diff --git a/src/prerna/reactor/automation/AutomationExecutionUtils.java b/src/prerna/reactor/automation/AutomationExecutionUtils.java index 65133be395d..2d4fca05591 100644 --- a/src/prerna/reactor/automation/AutomationExecutionUtils.java +++ b/src/prerna/reactor/automation/AutomationExecutionUtils.java @@ -155,7 +155,7 @@ public static String serializeRaw(Object rawResult) { @SuppressWarnings("unchecked") private static String transformRowsAsObjects(String rawStr) { - Map data = extractDataset(parseJson(rawStr)); + Map data = extractDataset(parseJsonAny(rawStr)); if (data == null) return rawStr; List headers = (List) data.get("headers"); List> rows = (List>) data.get("values"); @@ -173,7 +173,7 @@ private static String transformRowsAsObjects(String rawStr) { @SuppressWarnings("unchecked") private static String transformFirstRow(String rawStr) { - Map data = extractDataset(parseJson(rawStr)); + Map data = extractDataset(parseJsonAny(rawStr)); if (data == null) return rawStr; List headers = (List) data.get("headers"); List> rows = (List>) data.get("values"); @@ -189,7 +189,7 @@ private static String transformFirstRow(String rawStr) { @SuppressWarnings("unchecked") private static String transformColumn(String rawStr, String colName) { if (colName == null || colName.isEmpty()) return rawStr; - Map data = extractDataset(parseJson(rawStr)); + Map data = extractDataset(parseJsonAny(rawStr)); if (data == null) return rawStr; List headers = (List) data.get("headers"); List> rows = (List>) data.get("values"); @@ -205,7 +205,7 @@ private static String transformColumn(String rawStr, String colName) { private static String transformJsonPath(String rawStr, String path) { if (path == null || path.isEmpty()) return rawStr; try { - Object current = parseJson(rawStr); + Object current = parseJsonAny(rawStr); for (String segment : path.split("\\.")) { if (!(current instanceof Map)) break; current = ((Map) current).get(segment); @@ -217,16 +217,60 @@ private static String transformJsonPath(String rawStr, String path) { } } + /** + * Normalises any of the three dataset formats into a canonical {@code {headers, values}} map: + * 1. List<Map> (rows-as-objects) — produced by {@link DatabaseEngineNodeExecutor} + * 2. {@code {data: {headers, values}}} — SEMOSS wrapped envelope + * 3. {@code {headers, values}} — SEMOSS direct + */ @SuppressWarnings("unchecked") - private static Map extractDataset(Map parsed) { + private static Map extractDataset(Object parsed) { if (parsed == null) return null; - if (parsed.containsKey("data") && parsed.get("data") instanceof Map) { - return (Map) parsed.get("data"); + + // Format 1: rows-as-objects list from DatabaseEngineNodeExecutor + if (parsed instanceof List) { + List list = (List) parsed; + if (list.isEmpty()) return null; + Object first = list.get(0); + if (!(first instanceof Map)) return null; + List headers = new ArrayList<>(((Map) first).keySet()); + List> values = new ArrayList<>(); + for (Object item : list) { + if (item instanceof Map) { + Map row = (Map) item; + List rowVals = new ArrayList<>(); + for (String h : headers) rowVals.add(row.get(h)); + values.add(rowVals); + } + } + Map result = new HashMap<>(); + result.put("headers", headers); + result.put("values", values); + return result; + } + + if (!(parsed instanceof Map)) return null; + Map map = (Map) parsed; + + // Format 2: {data: {headers, values}} + if (map.containsKey("data") && map.get("data") instanceof Map) { + return (Map) map.get("data"); } - if (parsed.containsKey("headers") && parsed.containsKey("values")) return parsed; + // Format 3: {headers, values} + if (map.containsKey("headers") && map.containsKey("values")) return map; return null; } + /** Parses JSON to Object — returns List for arrays, Map for objects (handles all executor output shapes). */ + private static Object parseJsonAny(String json) { + if (json == null || json.isBlank()) return null; + try { + return GSON.fromJson(json, Object.class); + } catch (Exception e) { + return null; + } + } + private static Map parseJson(String json) { if (json == null || json.isBlank()) return null; try { From 82c7cfe4abbc8cd5d85b01ebd726de6b07191e53 Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Mon, 27 Jul 2026 17:21:47 -0400 Subject: [PATCH 08/25] refactor: extract run engine and add logging and standardization --- .../automation/AutomationConstants.java | 2 +- .../automation/AutomationExecutionUtils.java | 24 ++ .../automation/AutomationRunEngine.java | 247 +++++++++++++++++ .../CancelAutomationRunReactor.java | 8 +- .../automation/GetAutomationReactor.java | 18 +- .../automation/RunAutomationNodeReactor.java | 54 ++-- .../automation/TriggerAutomationReactor.java | 260 ++---------------- .../nodes/AutomationNodeContext.java | 13 +- .../nodes/DatabaseEngineNodeExecutor.java | 74 ++--- .../nodes/FunctionEngineNodeExecutor.java | 11 +- .../nodes/ModelEngineNodeExecutor.java | 11 +- .../nodes/StorageEngineNodeExecutor.java | 6 + .../nodes/VectorEngineNodeExecutor.java | 6 + .../automation/nodes/WaitNodeExecutor.java | 5 + 14 files changed, 413 insertions(+), 326 deletions(-) create mode 100644 src/prerna/reactor/automation/AutomationRunEngine.java diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index 978302f6fd0..15409f3d392 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -27,7 +27,7 @@ *******************************************************************************/ package prerna.reactor.automation; -public class AutomationConstants { +public final class AutomationConstants { private AutomationConstants() {} diff --git a/src/prerna/reactor/automation/AutomationExecutionUtils.java b/src/prerna/reactor/automation/AutomationExecutionUtils.java index 2d4fca05591..eba2784f601 100644 --- a/src/prerna/reactor/automation/AutomationExecutionUtils.java +++ b/src/prerna/reactor/automation/AutomationExecutionUtils.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.time.Instant; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedList; @@ -280,6 +281,29 @@ private static Map parseJson(String json) { } } + // -- Scope building ------------------------------------------------------------ + + /** + * Builds the initial variable scope for an automation run, seeded with {@code date}, + * {@code triggered_at}, and {@code run_id} (when non-blank). + */ + public static Map buildInitialScope(String runId) { + Map scope = new HashMap<>(); + // TODO: use user's session timezone instead of UTC when the Insight timezone API is finalized + String now = Instant.now().toString(); + scope.put("date", now.substring(0, 10)); + scope.put("triggered_at", now); + if (runId != null && !runId.isBlank()) scope.put("run_id", runId); + return scope; + } + + /** Truncates a string to {@link AutomationConstants#OUTPUT_PREVIEW_MAX_LENGTH} chars. */ + public static String generatePreview(String s) { + if (s == null) return null; + return s.length() <= AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH + ? s : s.substring(0, AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH); + } + // -- Config value coercion ----------------------------------------------------- /** diff --git a/src/prerna/reactor/automation/AutomationRunEngine.java b/src/prerna/reactor/automation/AutomationRunEngine.java new file mode 100644 index 00000000000..874f388230f --- /dev/null +++ b/src/prerna/reactor/automation/AutomationRunEngine.java @@ -0,0 +1,247 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.sql.Timestamp; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.om.Insight; +import prerna.reactor.automation.nodes.AutomationNodeContext; +import prerna.reactor.automation.nodes.AutomationNodeExecutors; +import prerna.reactor.automation.nodes.IAutomationNodeExecutor; +import prerna.util.Utility; + +/** + * Executes an automation run on a background thread. Separated from + * {@link TriggerAutomationReactor} so the reactor stays thin — it only parses params, + * claims a run slot, submits to the thread pool, and returns the run ID immediately. + */ +public final class AutomationRunEngine { + + private static final Logger classLogger = LogManager.getLogger(AutomationRunEngine.class); + + /** In-memory cancellation flags — fast path; the DB flag is the cluster-safe source of truth. */ + static final ConcurrentHashMap CANCELLATION_FLAGS = new ConcurrentHashMap<>(); + + private AutomationRunEngine() {} + + /** + * In-memory same-pod fast path for cancellation. + * Called by {@link CancelAutomationRunReactor}. + */ + public static boolean requestCancellation(String runId) { + AtomicBoolean flag = CANCELLATION_FLAGS.get(runId); + if (flag != null) { + flag.set(true); + return true; + } + return false; + } + + /** + * Runs the full automation node list, blocking until all nodes complete or the run is + * cancelled/failed. Must be called from a background thread (the automation executor). + * + * @param runId the run record ID (already inserted into DB by the caller) + * @param projectId the owning project + * @param ordered topologically sorted node list + * @param configMap project automation config key→value pairs + * @param insight the caller's insight context (propagated to each node executor) + */ + public static void run(String runId, String projectId, + List> ordered, Map configMap, Insight insight) { + + AtomicBoolean cancelled = new AtomicBoolean(false); + CANCELLATION_FLAGS.put(runId, cancelled); + ScheduledExecutorService heartbeat = startHeartbeat(runId); + + Map scope = AutomationExecutionUtils.buildInitialScope(runId); + int completedCount = 0; + + try { + for (Map node : ordered) { + String nodeId = (String) node.get("id"); + String nodeLabel = (String) node.get("label"); + String outputVar = (String) node.get("outputVar"); + String nodeType = (String) node.get("type"); + + if (cancelled.get() || AutomationDatabaseUtility.isCancelRequested(runId)) { + classLogger.info("Automation run {} cancelled before node {} ({})", runId, nodeId, nodeLabel); + AutomationDatabaseUtility.updateRunStatus(runId, + AutomationConstants.STATUS_CANCELLED, nodeId, "Run cancelled by user"); + return; + } + + Map nodeResult; + try { + nodeResult = executeSingleNode(runId, projectId, node, scope, configMap, cancelled, insight); + } catch (AutomationCancelledException ace) { + classLogger.info("Automation run {} cancelled during node {} ({})", runId, nodeId, nodeLabel); + AutomationDatabaseUtility.updateRunStatus(runId, + AutomationConstants.STATUS_CANCELLED, nodeId, ace.getMessage()); + return; + } + + String status = (String) nodeResult.get(AutomationConstants.STATUS); + + if (AutomationConstants.NODE_STATUS_SUCCESS.equals(status)) { + if (outputVar != null && !outputVar.isEmpty() + && !AutomationConstants.NODE_TRIGGER.equals(nodeType)) { + String outputValue = (String) nodeResult.get("outputValue"); + scope.put(outputVar, outputValue != null ? outputValue : ""); + } + completedCount++; + AutomationDatabaseUtility.updateHeartbeat(runId, completedCount); + } else { + String errorMsg = (String) nodeResult.get(AutomationConstants.ERROR_MESSAGE); + classLogger.warn("Automation run {} failed at node {} ({}): {}", runId, nodeId, nodeLabel, errorMsg); + AutomationDatabaseUtility.updateRunStatus(runId, + AutomationConstants.STATUS_FAILED, nodeId, errorMsg); + return; + } + } + + classLogger.info("Automation run {} completed successfully ({}/{} nodes)", runId, completedCount, ordered.size()); + AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_SUCCESS, null, null); + + } finally { + heartbeat.shutdownNow(); + CANCELLATION_FLAGS.remove(runId); + AutomationDatabaseUtility.releaseActiveRun(projectId, runId); + } + } + + // -- Node execution ------------------------------------------------------------ + + private static Map executeSingleNode(String runId, String projectId, + Map node, Map scope, Map configMap, + AtomicBoolean cancelFlag, Insight insight) { + + String nodeId = (String) node.get("id"); + String nodeLabel = (String) node.get("label"); + String outputVar = (String) node.get("outputVar"); + String type = (String) node.get("type"); + + if (AutomationConstants.NODE_TRIGGER.equals(type)) { + return buildNodeResult(nodeId, nodeLabel, AutomationConstants.NODE_STATUS_SUCCESS, 0, + scope.get("triggered_at"), null); + } + + classLogger.debug("Executing node {} ({}) type={} in run {}", nodeId, nodeLabel, type, runId); + AutomationDatabaseUtility.markNodeRunning(runId, nodeId); + Timestamp startedAt = toTimestamp(Instant.now()); + long startMs = System.currentTimeMillis(); + + try { + AutomationNodeContext ctx = new AutomationNodeContext( + runId, projectId, node, scope, configMap, insight, cancelFlag); + + IAutomationNodeExecutor executor = AutomationNodeExecutors.EXECUTORS.get(type); + if (executor == null) { + throw new IllegalArgumentException("Unsupported node type: " + type); + } + Object rawOutput = executor.execute(ctx); + + @SuppressWarnings("unchecked") + Map transformConfig = (Map) node.get("outputTransform"); + String transformed = AutomationExecutionUtils.applyOutputTransform(rawOutput, transformConfig); + long durationMs = System.currentTimeMillis() - startMs; + String preview = AutomationExecutionUtils.generatePreview(transformed); + + AutomationDatabaseUtility.updateNodeSuccess(runId, nodeId, startedAt, durationMs, outputVar, transformed, preview); + + classLogger.debug("Node {} ({}) succeeded in {}ms in run {}", nodeId, nodeLabel, durationMs, runId); + Map result = buildNodeResult(nodeId, nodeLabel, + AutomationConstants.NODE_STATUS_SUCCESS, durationMs, preview, null); + result.put("outputValue", transformed); + return result; + + } catch (AutomationCancelledException ace) { + long durationMs = System.currentTimeMillis() - startMs; + AutomationDatabaseUtility.updateNodeFailed(runId, nodeId, startedAt, durationMs, ace.getMessage()); + throw ace; + } catch (Exception e) { + long durationMs = System.currentTimeMillis() - startMs; + String errorMsg = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); + classLogger.error("Node {} ({}) failed in run {}: {}", nodeId, nodeLabel, runId, errorMsg, e); + AutomationDatabaseUtility.updateNodeFailed(runId, nodeId, startedAt, durationMs, errorMsg); + return buildNodeResult(nodeId, nodeLabel, + AutomationConstants.NODE_STATUS_FAILED, durationMs, null, errorMsg); + } + } + + // -- Heartbeat ----------------------------------------------------------------- + + private static ScheduledExecutorService startHeartbeat(String runId) { + ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "automation-heartbeat-" + runId.substring(0, 8)); + t.setDaemon(true); + return t; + }); + scheduler.scheduleAtFixedRate(() -> { + try { + AutomationDatabaseUtility.touchHeartbeat(runId); + } catch (Exception e) { + classLogger.warn("Heartbeat update failed for run {}: {}", runId, e.getMessage()); + } + }, AutomationConstants.HEARTBEAT_INTERVAL_SECONDS, + AutomationConstants.HEARTBEAT_INTERVAL_SECONDS, TimeUnit.SECONDS); + return scheduler; + } + + // -- Helpers ------------------------------------------------------------------- + + private static Map buildNodeResult(String nodeId, String nodeLabel, + String status, long durationMs, String outputPreview, String errorMessage) { + Map result = new HashMap<>(); + result.put(AutomationConstants.NODE_ID, nodeId); + result.put(AutomationConstants.NODE_LABEL, nodeLabel); + result.put(AutomationConstants.STATUS, status); + result.put(AutomationConstants.DURATION_MS, durationMs); + if (outputPreview != null) result.put(AutomationConstants.OUTPUT_PREVIEW, outputPreview); + if (errorMessage != null) result.put(AutomationConstants.ERROR_MESSAGE, errorMessage); + return result; + } + + private static Timestamp toTimestamp(Instant instant) { + return Utility.getSqlTimestampUTC(LocalDateTime.ofInstant(instant, ZoneOffset.UTC)); + } +} diff --git a/src/prerna/reactor/automation/CancelAutomationRunReactor.java b/src/prerna/reactor/automation/CancelAutomationRunReactor.java index fafb84fc620..df1bb097f65 100644 --- a/src/prerna/reactor/automation/CancelAutomationRunReactor.java +++ b/src/prerna/reactor/automation/CancelAutomationRunReactor.java @@ -48,7 +48,7 @@ *

Sets a cluster-safe cancellation flag ({@code AUTOMATION_RUNS.CANCEL_REQUESTED}, via * {@link AutomationDatabaseUtility#setCancelRequested(String)}) that the executing pod's * between-node check polls regardless of which pod that is - this is the source of truth. Also - * attempts an in-memory same-pod signal ({@link TriggerAutomationReactor#requestCancellation(String)}) + * attempts an in-memory same-pod signal ({@link AutomationRunEngine#requestCancellation(String)}) * as a fast path when the run happens to be executing on the pod that received this request. The * run's {@code STATUS} is transitioned to CANCELLED by whichever pod is actually executing it, * not by this reactor - a truly orphaned run (no pod executing it) is instead caught by the @@ -59,8 +59,8 @@ public class CancelAutomationRunReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(CancelAutomationRunReactor.class); public CancelAutomationRunReactor() { - this.keysToGet = new String[]{ "project", "runId" }; - this.keyRequired = new int[]{ 1, 1 }; + this.keysToGet = new String[] { "project", "runId" }; + this.keyRequired = new int[] { 1, 1 }; } @Override @@ -103,7 +103,7 @@ public NounMetadata execute() { // would be a lie. The executing pod transitions STATUS to CANCELLED itself once it // observes the flag; a truly orphaned run (crashed, nobody polling) is caught by the // periodic stale-heartbeat sweep (AutomationDatabaseUtility.markStaleRunsInterrupted). - boolean signalledLocally = TriggerAutomationReactor.requestCancellation(runId); + boolean signalledLocally = AutomationRunEngine.requestCancellation(runId); AutomationDatabaseUtility.setCancelRequested(runId); classLogger.info("Cancel requested for automation run {}: signalledLocally={}", runId, signalledLocally); diff --git a/src/prerna/reactor/automation/GetAutomationReactor.java b/src/prerna/reactor/automation/GetAutomationReactor.java index 36c3fde2165..a2625004b09 100644 --- a/src/prerna/reactor/automation/GetAutomationReactor.java +++ b/src/prerna/reactor/automation/GetAutomationReactor.java @@ -38,8 +38,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; import prerna.auth.utils.SecurityProjectUtils; @@ -51,13 +49,22 @@ import prerna.util.AssetUtility; import prerna.util.Utility; +/** + * Returns the automation definition (the saved {@code automation.json} graph) for a project. + * Returns an empty graph document when no automation has been saved yet. + * + *

Not to be confused with {@link GetAutomationRunReactor}, which returns + * run history records from the scheduler DB (AUTOMATION_RUNS + AUTOMATION_NODE_OUTPUTS). + * This reactor reads static config; that reactor reads live execution state. + * + *

Pixel: {@code GetAutomation(project=["appId"])} + */ public class GetAutomationReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(GetAutomationReactor.class); - private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); public GetAutomationReactor() { - this.keysToGet = new String[]{ "project" }; + this.keysToGet = new String[] { "project" }; } @Override @@ -95,7 +102,8 @@ public NounMetadata execute() { try { String json = Files.readString(automationFile.toPath(), StandardCharsets.UTF_8); - Map doc = GSON.fromJson(json, new TypeToken>() {}.getType()); + Map doc = AutomationExecutionUtils.GSON.fromJson(json, + new TypeToken>() {}.getType()); return new NounMetadata(doc, PixelDataType.MAP, PixelOperationType.OPERATION); } catch (IOException e) { classLogger.error("Error reading automation JSON", e); diff --git a/src/prerna/reactor/automation/RunAutomationNodeReactor.java b/src/prerna/reactor/automation/RunAutomationNodeReactor.java index 8e925d78928..61df2566069 100644 --- a/src/prerna/reactor/automation/RunAutomationNodeReactor.java +++ b/src/prerna/reactor/automation/RunAutomationNodeReactor.java @@ -27,11 +27,6 @@ *******************************************************************************/ package prerna.reactor.automation; -import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.time.Instant; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -40,10 +35,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.reflect.TypeToken; - import prerna.auth.utils.SecurityProjectUtils; import prerna.reactor.AbstractReactor; import prerna.reactor.automation.nodes.AutomationNodeContext; @@ -52,22 +43,20 @@ import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; import prerna.sablecc2.om.nounmeta.NounMetadata; -import prerna.util.AssetUtility; /** * Executes a single automation node for testing/preview purposes. * Result is NOT persisted to any run record. * - * Pixel: {@code RunAutomationNode(project=["appId"], nodeId=["node-id"], runId=["optional-context-run"])} + *

Pixel: {@code RunAutomationNode(project=["appId"], nodeId=["node-id"], runId=["optional-context-run"])} */ public class RunAutomationNodeReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(RunAutomationNodeReactor.class); - private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); public RunAutomationNodeReactor() { - this.keysToGet = new String[]{ "project", "nodeId", "runId" }; - this.keyRequired = new int[]{ 1, 1, 0 }; + this.keysToGet = new String[] { "project", "nodeId", "runId" }; + this.keyRequired = new int[] { 1, 1, 0 }; } @Override @@ -119,8 +108,7 @@ public NounMetadata execute() { Map transformConfig = (Map) node.get("outputTransform"); String transformed = AutomationExecutionUtils.applyOutputTransform(rawOutput, transformConfig); long durationMs = System.currentTimeMillis() - startMs; - String preview = (transformed != null && transformed.length() > AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH) - ? transformed.substring(0, AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH) : transformed; + String preview = AutomationExecutionUtils.generatePreview(transformed); Map result = new HashMap<>(); result.put(AutomationConstants.NODE_ID, nodeId); @@ -144,33 +132,20 @@ public NounMetadata execute() { } @SuppressWarnings("unchecked") - private Map findNode(String projectId, String nodeId) { - String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); - File f = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); - if (!f.exists()) { - throw new IllegalArgumentException("No automation.json found for this project"); - } - try { - String json = Files.readString(f.toPath(), StandardCharsets.UTF_8); - Map doc = GSON.fromJson(json, new TypeToken>() {}.getType()); - Map graph = (Map) doc.get("graph"); - List> nodes = (List>) graph.get("nodes"); - if (nodes != null) { - for (Map node : nodes) { - if (nodeId.equals(node.get("id"))) return node; - } + private static Map findNode(String projectId, String nodeId) { + Map doc = AutomationExecutionUtils.loadAutomationDoc(projectId); + Map graph = (Map) doc.get("graph"); + List> nodes = (List>) graph.get("nodes"); + if (nodes != null) { + for (Map node : nodes) { + if (nodeId.equals(node.get("id"))) return node; } - } catch (IOException e) { - throw new IllegalStateException("Failed to read automation.json: " + e.getMessage(), e); } return null; } private Map buildScope(String contextRunId) { - Map scope = new HashMap<>(); - String now = Instant.now().toString(); - scope.put("date", now.substring(0, 10)); - scope.put("triggered_at", now); + Map scope = AutomationExecutionUtils.buildInitialScope(null); if (contextRunId != null && !contextRunId.isEmpty()) { List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(contextRunId); @@ -187,4 +162,9 @@ private Map buildScope(String contextRunId) { } return scope; } + + @Override + public String getReactorDescription() { + return "Executes a single automation node in isolation for testing — result is not persisted."; + } } diff --git a/src/prerna/reactor/automation/TriggerAutomationReactor.java b/src/prerna/reactor/automation/TriggerAutomationReactor.java index 869006eb97d..b1e825adefb 100644 --- a/src/prerna/reactor/automation/TriggerAutomationReactor.java +++ b/src/prerna/reactor/automation/TriggerAutomationReactor.java @@ -27,25 +27,16 @@ *******************************************************************************/ package prerna.reactor.automation; -import java.sql.Timestamp; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -53,20 +44,20 @@ import prerna.auth.utils.SecurityProjectUtils; import prerna.om.ThreadStore; import prerna.reactor.AbstractReactor; -import prerna.reactor.automation.nodes.AutomationNodeContext; -import prerna.reactor.automation.nodes.AutomationNodeExecutors; -import prerna.reactor.automation.nodes.IAutomationNodeExecutor; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; import prerna.sablecc2.om.nounmeta.NounMetadata; -import prerna.util.Utility; +/** + * Manually triggers an automation run for a project. Validates access, claims the single-run slot, + * submits execution to a background thread pool, and returns the run ID immediately for polling. + * + *

Pixel: {@code TriggerAutomation(project=["appId"])} + */ public class TriggerAutomationReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(TriggerAutomationReactor.class); - private static final ConcurrentHashMap CANCELLATION_FLAGS = new ConcurrentHashMap<>(); - private static final ExecutorService AUTOMATION_EXECUTOR = new ThreadPoolExecutor( 2, 20, 60L, TimeUnit.SECONDS, new LinkedBlockingQueue<>(10), @@ -79,8 +70,8 @@ public class TriggerAutomationReactor extends AbstractReactor { ); public TriggerAutomationReactor() { - this.keysToGet = new String[]{ "project" }; - this.keyRequired = new int[]{ 1 }; + this.keysToGet = new String[] { "project" }; + this.keyRequired = new int[] { 1 }; } @Override @@ -107,6 +98,8 @@ public NounMetadata execute() { List> edges = (List>) graph.get("edges"); Map configMap = AutomationExecutionUtils.loadConfig(projectId); + // The form view always produces a sequential node list with no edges, so topoSort + // degrades to node-list order. Edges + sort are kept for a future visual flow editor. List> ordered = AutomationExecutionUtils.topoSort(nodes, edges); if (ordered.isEmpty()) { throw new IllegalArgumentException("Automation has no nodes to execute"); @@ -124,9 +117,9 @@ public NounMetadata execute() { AUTOMATION_EXECUTOR.submit(() -> { installThreadContext(contextSnapshot); try { - executeNodes(runId, projectId, ordered, configMap); + AutomationRunEngine.run(runId, projectId, ordered, configMap, this.insight); } catch (Exception e) { - classLogger.error("Unhandled error executing automation run {}: {}", runId, e.getMessage(), e); + classLogger.error("Unhandled error in automation run {}: {}", runId, e.getMessage(), e); AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_FAILED, null, e.getMessage()); } finally { @@ -139,175 +132,27 @@ public NounMetadata execute() { throw new IllegalStateException("Too many concurrent automation runs. Please try again shortly."); } - Map result = buildRunResult(runId, projectId, AutomationConstants.STATUS_RUNNING, - ordered.size(), 0, null, new ArrayList<>()); + classLogger.info("Automation run {} submitted for project {}", runId, projectId); + + Map stored = AutomationDatabaseUtility.getRunDetail(runId); + Map result = new HashMap<>(); + result.put(AutomationConstants.RUN_ID, runId); + result.put(AutomationConstants.PROJECT_ID, projectId); + result.put(AutomationConstants.STATUS, AutomationConstants.STATUS_RUNNING); + result.put(AutomationConstants.TOTAL_NODES, ordered.size()); + result.put(AutomationConstants.COMPLETED_NODES, 0); + if (stored != null) { + result.put(AutomationConstants.STARTED_AT, stored.get(AutomationConstants.STARTED_AT)); + } + result.put("nodeResults", new ArrayList<>()); return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } catch (RuntimeException e) { AutomationDatabaseUtility.releaseActiveRun(projectId, runId); throw e; } } - // -- Core Execution ------------------------------------------------------------ - - private Map executeNodes(String runId, String projectId, - List> ordered, Map configMap) { - - AtomicBoolean cancelled = new AtomicBoolean(false); - CANCELLATION_FLAGS.put(runId, cancelled); - - ScheduledExecutorService heartbeat = startHeartbeat(runId); - - Map scope = buildInitialScope(runId); - List> nodeResults = new ArrayList<>(); - int completedCount = 0; - - try { - for (Map node : ordered) { - String nodeId = (String) node.get("id"); - String nodeLabel = (String) node.get("label"); - String outputVar = (String) node.get("outputVar"); - String nodeType = (String) node.get("type"); - - if (cancelled.get() || AutomationDatabaseUtility.isCancelRequested(runId)) { - AutomationDatabaseUtility.updateRunStatus(runId, - AutomationConstants.STATUS_CANCELLED, nodeId, "Run cancelled by user"); - nodeResults.add(buildNodeResult(nodeId, nodeLabel, - AutomationConstants.STATUS_CANCELLED, 0, null, "Run cancelled by user")); - return buildRunResult(runId, projectId, AutomationConstants.STATUS_CANCELLED, - ordered.size(), completedCount, nodeId, nodeResults); - } - - Map nodeResult; - try { - nodeResult = executeSingleNode(runId, projectId, node, scope, configMap); - } catch (AutomationCancelledException ace) { - AutomationDatabaseUtility.updateRunStatus(runId, - AutomationConstants.STATUS_CANCELLED, nodeId, ace.getMessage()); - nodeResults.add(buildNodeResult(nodeId, nodeLabel, - AutomationConstants.NODE_STATUS_FAILED, 0, null, ace.getMessage())); - return buildRunResult(runId, projectId, AutomationConstants.STATUS_CANCELLED, - ordered.size(), completedCount, nodeId, nodeResults); - } - - String status = (String) nodeResult.get(AutomationConstants.STATUS); - nodeResults.add(nodeResult); - - if (AutomationConstants.NODE_STATUS_SUCCESS.equals(status)) { - if (outputVar != null && !outputVar.isEmpty() - && !AutomationConstants.NODE_TRIGGER.equals(nodeType)) { - String outputValue = (String) nodeResult.get("outputValue"); - scope.put(outputVar, outputValue != null ? outputValue : ""); - } - completedCount++; - AutomationDatabaseUtility.updateHeartbeat(runId, completedCount); - } else { - String errorMsg = (String) nodeResult.get(AutomationConstants.ERROR_MESSAGE); - AutomationDatabaseUtility.updateRunStatus(runId, - AutomationConstants.STATUS_FAILED, nodeId, errorMsg); - return buildRunResult(runId, projectId, AutomationConstants.STATUS_FAILED, - ordered.size(), completedCount, nodeId, nodeResults); - } - } - - AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_SUCCESS, null, null); - return buildRunResult(runId, projectId, AutomationConstants.STATUS_SUCCESS, - ordered.size(), completedCount, null, nodeResults); - - } finally { - heartbeat.shutdownNow(); - CANCELLATION_FLAGS.remove(runId); - AutomationDatabaseUtility.releaseActiveRun(projectId, runId); - } - } - - private Map executeSingleNode(String runId, String projectId, Map node, - Map scope, Map configMap) { - - String nodeId = (String) node.get("id"); - String nodeLabel = (String) node.get("label"); - String outputVar = (String) node.get("outputVar"); - String type = (String) node.get("type"); - - // Trigger node is a metadata-only node — just return success - if (AutomationConstants.NODE_TRIGGER.equals(type)) { - return buildNodeResult(nodeId, nodeLabel, AutomationConstants.NODE_STATUS_SUCCESS, 0, - scope.get("triggered_at"), null); - } - - AutomationDatabaseUtility.markNodeRunning(runId, nodeId); - Timestamp startedAt = toTimestamp(Instant.now()); - long startMs = System.currentTimeMillis(); - - try { - AtomicBoolean cancelFlag = CANCELLATION_FLAGS.get(runId); - AutomationNodeContext ctx = new AutomationNodeContext( - runId, projectId, node, scope, configMap, this.insight, cancelFlag); - - IAutomationNodeExecutor executor = AutomationNodeExecutors.EXECUTORS.get(type); - if (executor == null) { - throw new IllegalArgumentException("Unsupported node type: " + type); - } - Object rawOutput = executor.execute(ctx); - - @SuppressWarnings("unchecked") - Map transformConfig = (Map) node.get("outputTransform"); - String transformed = AutomationExecutionUtils.applyOutputTransform(rawOutput, transformConfig); - - long durationMs = System.currentTimeMillis() - startMs; - String preview = generatePreview(transformed); - - AutomationDatabaseUtility.updateNodeSuccess(runId, nodeId, startedAt, durationMs, outputVar, transformed, preview); - - Map result = buildNodeResult(nodeId, nodeLabel, - AutomationConstants.NODE_STATUS_SUCCESS, durationMs, preview, null); - result.put("outputValue", transformed); - return result; - - } catch (AutomationCancelledException ace) { - long durationMs = System.currentTimeMillis() - startMs; - AutomationDatabaseUtility.updateNodeFailed(runId, nodeId, startedAt, durationMs, ace.getMessage()); - throw ace; - } catch (Exception e) { - long durationMs = System.currentTimeMillis() - startMs; - String errorMsg = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); - classLogger.error("Node {} ({}) failed: {}", nodeId, nodeLabel, errorMsg, e); - AutomationDatabaseUtility.updateNodeFailed(runId, nodeId, startedAt, durationMs, errorMsg); - return buildNodeResult(nodeId, nodeLabel, - AutomationConstants.NODE_STATUS_FAILED, durationMs, null, errorMsg); - } - } - - // -- Heartbeat ----------------------------------------------------------------- - - private ScheduledExecutorService startHeartbeat(String runId) { - ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> { - Thread t = new Thread(r, "automation-heartbeat-" + runId.substring(0, 8)); - t.setDaemon(true); - return t; - }); - scheduler.scheduleAtFixedRate(() -> { - try { - AutomationDatabaseUtility.touchHeartbeat(runId); - } catch (Exception e) { - classLogger.warn("Heartbeat update failed for run {}: {}", runId, e.getMessage()); - } - }, AutomationConstants.HEARTBEAT_INTERVAL_SECONDS, - AutomationConstants.HEARTBEAT_INTERVAL_SECONDS, TimeUnit.SECONDS); - return scheduler; - } - - // -- Cancellation Support ------------------------------------------------------ - - public static boolean requestCancellation(String runId) { - AtomicBoolean flag = CANCELLATION_FLAGS.get(runId); - if (flag != null) { - flag.set(true); - return true; - } - return false; - } - // -- Helpers ------------------------------------------------------------------- private static void installThreadContext(Map snapshot) { @@ -335,55 +180,8 @@ private String getUserId() { return "system"; } - private Map buildInitialScope(String runId) { - Map scope = new HashMap<>(); - String now = Instant.now().toString(); - scope.put("date", now.substring(0, 10)); - scope.put("triggered_at", now); - scope.put("run_id", runId); - return scope; - } - - private Timestamp toTimestamp(Instant instant) { - return Utility.getSqlTimestampUTC(LocalDateTime.ofInstant(instant, ZoneOffset.UTC)); - } - - private static String generatePreview(String s) { - if (s == null) return null; - return s.length() <= AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH - ? s : s.substring(0, AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH); - } - - // -- Result Building ----------------------------------------------------------- - - private Map buildNodeResult(String nodeId, String nodeLabel, - String status, long durationMs, String outputPreview, String errorMessage) { - Map result = new HashMap<>(); - result.put(AutomationConstants.NODE_ID, nodeId); - result.put(AutomationConstants.NODE_LABEL, nodeLabel); - result.put(AutomationConstants.STATUS, status); - result.put(AutomationConstants.DURATION_MS, durationMs); - if (outputPreview != null) result.put(AutomationConstants.OUTPUT_PREVIEW, outputPreview); - if (errorMessage != null) result.put(AutomationConstants.ERROR_MESSAGE, errorMessage); - return result; - } - - private Map buildRunResult(String runId, String projectId, String status, - int totalNodes, int completedNodes, String failedNodeId, - List> nodeResults) { - Map stored = AutomationDatabaseUtility.getRunDetail(runId); - Map result = new HashMap<>(); - result.put(AutomationConstants.RUN_ID, runId); - result.put(AutomationConstants.PROJECT_ID, projectId); - result.put(AutomationConstants.STATUS, status); - result.put(AutomationConstants.TOTAL_NODES, totalNodes); - result.put(AutomationConstants.COMPLETED_NODES, completedNodes); - if (stored != null) { - result.put(AutomationConstants.STARTED_AT, stored.get(AutomationConstants.STARTED_AT)); - result.put(AutomationConstants.COMPLETED_AT, stored.get(AutomationConstants.COMPLETED_AT)); - } - if (failedNodeId != null) result.put(AutomationConstants.FAILED_NODE_ID, failedNodeId); - result.put("nodeResults", nodeResults); - return result; + @Override + public String getReactorDescription() { + return "Manually triggers an automation run for the given project and returns a run ID for polling."; } } diff --git a/src/prerna/reactor/automation/nodes/AutomationNodeContext.java b/src/prerna/reactor/automation/nodes/AutomationNodeContext.java index 165e5d04bb7..0ae3fbeeafa 100644 --- a/src/prerna/reactor/automation/nodes/AutomationNodeContext.java +++ b/src/prerna/reactor/automation/nodes/AutomationNodeContext.java @@ -33,7 +33,18 @@ import prerna.om.Insight; /** - * Single param object bundling everything an {@link IAutomationNodeExecutor} needs to run one node. + * Immutable param bundle passed to every {@link IAutomationNodeExecutor}. + * + *

Consolidates all per-node execution inputs so executor implementations have a single, + * typed object to work with rather than long parameter lists. + * + * @param runId the automation run ID (use {@code "test"} for single-node test runs) + * @param projectId the owning project UUID + * @param node the raw node definition map from {@code automation.json} + * @param scope mutable variable scope — string key→value pairs resolved by {@code ${varName}} + * @param configMap project automation config key→value pairs resolved by {@code ${config.KEY}} + * @param insight caller's {@link Insight} context (carries user, session, pixel engine) + * @param cancelFlag shared flag the executor checks to honour mid-node cancellation requests */ public record AutomationNodeContext( String runId, diff --git a/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java index 66ca23d4684..8274a18d92e 100644 --- a/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java @@ -27,22 +27,31 @@ *******************************************************************************/ package prerna.reactor.automation.nodes; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; -import prerna.engine.api.IRDBMSEngine; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import prerna.reactor.automation.AutomationExecutionUtils; -import prerna.util.Utility; +import prerna.reactor.automation.PixelExecutionUtils; +/** + * Executor for {@code database-engine} nodes. Runs a SQL query against a configured database engine + * using the {@code SqlQuery} reactor — the same engine abstraction used elsewhere in the platform. + * + *

Config fields: + *

    + *
  • {@code engineId} (required) — UUID or alias of the target database engine
    • + *
  • {@code expression} (required) — SQL expression; supports {@code ${var}} substitution
    • + *
  • {@code operation} (optional) — {@code "read"} (default) or {@code "write"}; informational + * only since {@code SqlQuery} auto-detects SELECT vs DML from the SQL text
    • + *
  • {@code limit} (optional) — max rows for SELECT results; default 50
    • + *
+ */ public final class DatabaseEngineNodeExecutor implements IAutomationNodeExecutor { + private static final Logger classLogger = LogManager.getLogger(DatabaseEngineNodeExecutor.class); + @Override public Object execute(AutomationNodeContext ctx) throws Exception { Map config = ctx.config(); @@ -53,46 +62,21 @@ public Object execute(AutomationNodeContext ctx) throws Exception { String engineId = required(config, "engineId", nodeLabel); String sql = required(config, "expression", nodeLabel); String operation = optional(config, "operation", "read"); + int limit = optionalInt(config, "limit", 50); String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); String resolvedSql = AutomationExecutionUtils.resolve(sql, scope, configMap); - IRDBMSEngine engine = (IRDBMSEngine) Utility.getEngine(resolvedEngineId); - if (engine == null) { - throw new IllegalArgumentException("Database-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); - } + classLogger.debug("Database-engine node \"{}\" executing operation={} via engine {}", nodeLabel, operation, resolvedEngineId); - if ("write".equals(operation)) { - try (Connection conn = engine.getConnection(); - PreparedStatement ps = conn.prepareStatement(resolvedSql)) { - int rowsAffected = ps.executeUpdate(); - return Map.of("rowsAffected", rowsAffected); - } catch (SQLException e) { - throw new IllegalStateException("Database-engine node \"" + nodeLabel + "\": write failed: " + e.getMessage(), e); - } - } else { - int limit = optionalInt(config, "limit", 50); - try (Connection conn = engine.getConnection(); - PreparedStatement ps = conn.prepareStatement(resolvedSql)) { - try (ResultSet rs = ps.executeQuery()) { - ResultSetMetaData meta = rs.getMetaData(); - int colCount = meta.getColumnCount(); - List> rows = new ArrayList<>(); - int count = 0; - while (rs.next() && count < limit) { - Map row = new LinkedHashMap<>(); - for (int i = 1; i <= colCount; i++) { - row.put(meta.getColumnLabel(i), rs.getObject(i)); - } - rows.add(row); - count++; - } - return rows; - } - } catch (SQLException e) { - throw new IllegalStateException("Database-engine node \"" + nodeLabel + "\": query failed: " + e.getMessage(), e); - } - } + // Escape double quotes in the SQL for the pixel string literal, then delegate to + // SqlQuery which uses the engine abstraction (HardSelectQueryStruct) and enforces + // the caller's database-level permissions automatically. + String escapedSql = resolvedSql.replace("\"", "\\\""); + String pixel = "SqlQuery(database=[\"" + resolvedEngineId + "\"], query=[\"" + escapedSql + "\"], limit=[" + limit + "]);"; + + int timeout = AutomationExecutionUtils.getNodeTimeout(ctx.node()); + return PixelExecutionUtils.runAndCollect(ctx.insight(), pixel, timeout); } private static String required(Map config, String key, String nodeLabel) { diff --git a/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java index 500c5c11e3e..c1c0928063a 100644 --- a/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java @@ -29,12 +29,19 @@ import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.reflect.TypeToken; + import prerna.engine.api.IFunctionEngine; import prerna.reactor.automation.AutomationExecutionUtils; import prerna.util.Utility; public final class FunctionEngineNodeExecutor implements IAutomationNodeExecutor { + private static final Logger classLogger = LogManager.getLogger(FunctionEngineNodeExecutor.class); + @Override public Object execute(AutomationNodeContext ctx) throws Exception { Map config = ctx.config(); @@ -53,6 +60,7 @@ public Object execute(AutomationNodeContext ctx) throws Exception { throw new IllegalArgumentException("Function-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); } + classLogger.debug("Function-engine node \"{}\" executing via engine {}", nodeLabel, resolvedEngineId); Map paramMap = parseParams(resolvedParams, nodeLabel); return engine.execute(paramMap); } @@ -61,7 +69,8 @@ public Object execute(AutomationNodeContext ctx) throws Exception { private static Map parseParams(String json, String nodeLabel) { if (json == null || json.isBlank()) return Map.of(); try { - Map parsed = AutomationExecutionUtils.GSON.fromJson(json, Map.class); + Map parsed = AutomationExecutionUtils.GSON.fromJson(json, + new TypeToken>() {}.getType()); return parsed != null ? parsed : Map.of(); } catch (Exception e) { throw new IllegalArgumentException("Function-engine node \"" + nodeLabel + "\": params is not valid JSON: " + e.getMessage(), e); diff --git a/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java index 8df09d75d29..2c4eeec20a1 100644 --- a/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java @@ -31,6 +31,11 @@ import java.util.List; import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.reflect.TypeToken; + import prerna.engine.api.IModelEngine; import prerna.engine.impl.model.responses.AskModelEngineResponse; import prerna.engine.impl.model.responses.EmbeddingsModelEngineResponse; @@ -39,6 +44,8 @@ public final class ModelEngineNodeExecutor implements IAutomationNodeExecutor { + private static final Logger classLogger = LogManager.getLogger(ModelEngineNodeExecutor.class); + @Override public Object execute(AutomationNodeContext ctx) throws Exception { Map config = ctx.config(); @@ -55,6 +62,7 @@ public Object execute(AutomationNodeContext ctx) throws Exception { throw new IllegalArgumentException("Model-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); } + classLogger.debug("Model-engine node \"{}\" executing operation={} via engine {}", nodeLabel, operation, resolvedEngineId); switch (operation) { case "embeddings": { String values = required(config, "values", nodeLabel); @@ -85,7 +93,8 @@ private static Map parseParams(Map config, if (paramValues == null) return null; String resolved = AutomationExecutionUtils.resolve(paramValues, scope, configMap); try { - return AutomationExecutionUtils.GSON.fromJson(resolved, Map.class); + return AutomationExecutionUtils.GSON.fromJson(resolved, + new TypeToken>() {}.getType()); } catch (Exception e) { throw new IllegalArgumentException("Model-engine node \"" + nodeLabel + "\": paramValues is not valid JSON: " + e.getMessage(), e); } diff --git a/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java index cdb7a145b36..578e02d5a4f 100644 --- a/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java @@ -31,12 +31,17 @@ import java.util.List; import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import prerna.engine.api.IStorageEngine; import prerna.reactor.automation.AutomationExecutionUtils; import prerna.util.Utility; public final class StorageEngineNodeExecutor implements IAutomationNodeExecutor { + private static final Logger classLogger = LogManager.getLogger(StorageEngineNodeExecutor.class); + @Override public Object execute(AutomationNodeContext ctx) throws Exception { Map config = ctx.config(); @@ -53,6 +58,7 @@ public Object execute(AutomationNodeContext ctx) throws Exception { throw new IllegalArgumentException("Storage-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); } + classLogger.debug("Storage-engine node \"{}\" executing operation={} via engine {}", nodeLabel, operation, resolvedEngineId); switch (operation) { case "download": { String storagePath = required(config, "storagePath", nodeLabel); diff --git a/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java index 4006a28cd85..459f6c988e1 100644 --- a/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java @@ -31,12 +31,17 @@ import java.util.List; import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import prerna.engine.api.IVectorDatabaseEngine; import prerna.reactor.automation.AutomationExecutionUtils; import prerna.util.Utility; public final class VectorEngineNodeExecutor implements IAutomationNodeExecutor { + private static final Logger classLogger = LogManager.getLogger(VectorEngineNodeExecutor.class); + @Override public Object execute(AutomationNodeContext ctx) throws Exception { Map config = ctx.config(); @@ -53,6 +58,7 @@ public Object execute(AutomationNodeContext ctx) throws Exception { throw new IllegalArgumentException("Vector-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); } + classLogger.debug("Vector-engine node \"{}\" executing operation={} via engine {}", nodeLabel, operation, resolvedEngineId); switch (operation) { case "add-file": case "add-csv": { diff --git a/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java index be309caddd8..ba55007219a 100644 --- a/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java @@ -30,6 +30,9 @@ import java.util.Map; import java.util.concurrent.TimeUnit; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import prerna.reactor.automation.AutomationCancelledException; import prerna.reactor.automation.AutomationDatabaseUtility; import prerna.reactor.automation.AutomationExecutionUtils; @@ -46,6 +49,7 @@ */ public final class WaitNodeExecutor implements IAutomationNodeExecutor { + private static final Logger classLogger = LogManager.getLogger(WaitNodeExecutor.class); private static final int CANCEL_CHECK_INTERVAL_SECONDS = 5; @Override @@ -65,6 +69,7 @@ public Object execute(AutomationNodeContext ctx) throws Exception { "\" - seconds value is not a valid integer after resolution: \"" + resolved + "\""); } seconds = Math.min(Math.max(seconds, 0), 3600); + classLogger.debug("Wait node \"{}\" sleeping {} seconds", nodeLabel, seconds); // Sleep in chunks so cancel requests are honored mid-wait rather than waiting // for the full duration to complete. From 7e8b846a2d549bb20f8bd3e1f381332dc7c92c41 Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Tue, 28 Jul 2026 14:27:59 -0400 Subject: [PATCH 09/25] chore: add more automation documentation --- src/prerna/reactor/automation/AGENTS.md | 67 +++++++++++++++++++ .../automation/GetAutomationReactor.java | 12 +++- src/prerna/reactor/automation/README.md | 65 ++++++++++++++++++ .../nodes/AppEngineNodeExecutor.java | 7 ++ 4 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 src/prerna/reactor/automation/AGENTS.md create mode 100644 src/prerna/reactor/automation/README.md diff --git a/src/prerna/reactor/automation/AGENTS.md b/src/prerna/reactor/automation/AGENTS.md new file mode 100644 index 00000000000..79c3e918323 --- /dev/null +++ b/src/prerna/reactor/automation/AGENTS.md @@ -0,0 +1,67 @@ +# Automation Engine — Agent Guide + +Read `README.md` first for the execution model and DB schema. + +## Adding a new node type + +1. **Create the executor** in `nodes/` implementing `IAutomationNodeExecutor`: + - Declare `private static final Logger classLogger = LogManager.getLogger(YourExecutor.class);` + - Log execution at `DEBUG` level with node label and key params before dispatching + - Use `AutomationExecutionUtils.resolve(value, scope, configMap)` for all `${var}` substitution + - Throw `IllegalArgumentException` for missing required config fields + - Use `AutomationExecutionUtils.GSON` — do not declare a local `Gson` instance + +2. **Register it** in `AutomationNodeExecutors.EXECUTORS` map + +3. **Add the type constant** to `AutomationConstants` (e.g. `NODE_TYPE_FOO = "foo-engine"`) + +4. **Wire the FE** — add the node type to `automation.types.ts` and `automation.constants.ts` in `SemossWeb` + +## Logging rules + +Follow the platform standard — SLF4J `{}` placeholders, exception as the last argument: + +```java +// ✅ +classLogger.debug("Foo node \"{}\" executing operation={}", nodeLabel, operation); +classLogger.error("Foo node \"{}\" failed: {}", nodeLabel, e.getMessage(), e); + +// ❌ — never concatenate strings in log calls +classLogger.error("Foo node " + nodeLabel + " failed: " + e.getMessage()); +``` + +## Exception conventions + +```java +// Missing/invalid user input — use IllegalArgumentException +throw new IllegalArgumentException("Foo node \"" + nodeLabel + "\": 'engineId' is required"); + +// User-facing reactor errors — use SemossPixelException +throw new SemossPixelException("Project does not exist or user does not have access"); +``` + +## GSON + +Use the shared instance — never declare your own: + +```java +// ✅ +AutomationExecutionUtils.GSON.fromJson(json, new TypeToken>() {}.getType()); + +// ❌ +private static final Gson GSON = new GsonBuilder().create(); +``` + +## Reactor conventions + +- Keep reactors thin: parse params, auth-check, delegate, return. Business logic belongs in `AutomationRunEngine`, `AutomationExecutionUtils`, or an executor. +- Always call `organizeKeys()` before reading `this.keyValue` +- Use `SecurityProjectUtils.testUserProjectIdForAlias` to resolve alias → UUID before any lookup +- Add `getReactorDescription()` to every reactor + +## What not to change + +- `AutomationDatabaseUtility` — DB access is intentional; use `setNullableString`, `SelectQueryStruct`, try-with-resources +- `PixelExecutionUtils` — timeout + ThreadStore propagation; do not bypass +- `CancelAutomationRunReactor` — dual-signal cancel (DB flag + in-memory); both signals are required for cluster safety +- `claimActiveRun` — PK-violation is the concurrency guard; do not add a separate lock diff --git a/src/prerna/reactor/automation/GetAutomationReactor.java b/src/prerna/reactor/automation/GetAutomationReactor.java index a2625004b09..c07ff49db41 100644 --- a/src/prerna/reactor/automation/GetAutomationReactor.java +++ b/src/prerna/reactor/automation/GetAutomationReactor.java @@ -53,9 +53,15 @@ * Returns the automation definition (the saved {@code automation.json} graph) for a project. * Returns an empty graph document when no automation has been saved yet. * - *

Not to be confused with {@link GetAutomationRunReactor}, which returns - * run history records from the scheduler DB (AUTOMATION_RUNS + AUTOMATION_NODE_OUTPUTS). - * This reactor reads static config; that reactor reads live execution state. + *

There are three "get" reactors — each reads something different: + *

    + *
  • {@code GetAutomation} (this reactor) — reads {@code automation.json}: the pipeline graph + * (nodes, edges, output transforms). Static config written by {@link SaveAutomationReactor}.
    • + *
  • {@link GetAutomationConfigReactor GetAutomationConfig} — reads {@code automation_config.json}: + * key/value env vars and secrets; sensitive values are masked in the response.
    • + *
  • {@link GetAutomationRunReactor GetAutomationRun} — reads live run state from the DB + * (AUTOMATION_RUNS + AUTOMATION_NODE_OUTPUTS); used by the FE to poll execution progress.
    • + *
* *

Pixel: {@code GetAutomation(project=["appId"])} */ diff --git a/src/prerna/reactor/automation/README.md b/src/prerna/reactor/automation/README.md new file mode 100644 index 00000000000..8a23e0759fc --- /dev/null +++ b/src/prerna/reactor/automation/README.md @@ -0,0 +1,65 @@ +# Automation Engine + +Executes sequential node pipelines against SEMOSS engines. Users build a pipeline in the form editor (FE), save it as `automation.json`, then trigger it manually. Each run is tracked in the DB and the FE polls for progress. + +## How it works + +``` +TriggerAutomationReactor + → reads automation.json (nodes in order) + → claims single-run slot (AUTOMATION_ACTIVE_RUN) + → submits to ThreadPoolExecutor (2–20 threads) + → returns runId immediately + +AutomationRunEngine (background thread) + → iterates nodes in saved order + → dispatches each node to its IAutomationNodeExecutor + → writes node output + status to AUTOMATION_NODE_OUTPUTS + → FE polls GetAutomationRunReactor every 3s +``` + +## Reactors + +| Reactor | Pixel | What it does | +| --- | --- | --- | +| `TriggerAutomationReactor` | `TriggerAutomation(project=["id"])` | Starts a run, returns runId | +| `GetAutomationReactor` | `GetAutomation(project=["id"])` | Returns saved pipeline definition (automation.json) | +| `GetAutomationConfigReactor` | `GetAutomationConfig(project=["id"])` | Returns env var/secret config; masks sensitive values | +| `GetAutomationRunReactor` | `GetAutomationRun(project=["id"], runId=["id"])` | Returns live run state for FE polling | +| `ListAutomationRunsReactor` | `ListAutomationRuns(project=["id"])` | Returns run history | +| `CancelAutomationRunReactor` | `CancelAutomationRun(project=["id"], runId=["id"])` | Cancels an in-progress run | +| `SaveAutomationReactor` | `SaveAutomation(project=["id"], config=["{}"])` | Persists pipeline definition | +| `SaveAutomationConfigReactor` | `SaveAutomationConfig(project=["id"], config=["[]"])` | Persists env var config | +| `RunAutomationNodeReactor` | `RunAutomationNode(project=["id"], nodeId=["id"])` | Single-node test run; result not persisted | + +## Node types + +Each node type has a corresponding `IAutomationNodeExecutor` in `nodes/`: + +| Type | Executor | What it does | +| --- | --- | --- | +| `trigger` | (no executor) | Seed node; provides `triggered_at`, `date`, `run_id` scope vars | +| `database-engine` | `DatabaseEngineNodeExecutor` | Runs SQL via `SqlQuery` pixel | +| `model-engine` | `ModelEngineNodeExecutor` | LLM ask or embeddings via `IModelEngine` | +| `vector-engine` | `VectorEngineNodeExecutor` | Search, add, delete, list via `IVectorDatabaseEngine` | +| `storage-engine` | `StorageEngineNodeExecutor` | File operations via `IStorageEngine` | +| `function-engine` | `FunctionEngineNodeExecutor` | Function invocation via `IFunctionEngine` | +| `app` | `AppEngineNodeExecutor` | Arbitrary pixel, optionally scoped to a project | +| `wait` | `WaitNodeExecutor` | Sleep N seconds; cancel-aware | + +## DB tables + +| Table | Purpose | +| --- | --- | +| `AUTOMATION_ACTIVE_RUN` | PK on `PROJECT_ID` — enforces one concurrent run per project | +| `AUTOMATION_RUNS` | One row per run: status, timing, node counts | +| `AUTOMATION_NODE_OUTPUTS` | One row per node per run: status, output, preview, duration | + +## Key utilities + +| Class | Purpose | +| --- | --- | +| `AutomationExecutionUtils` | Shared statics: GSON, scope building, variable resolution, output transforms, preview generation | +| `AutomationDatabaseUtility` | All DB reads/writes for runs and node outputs | +| `PixelExecutionUtils` | Timeout-enforced pixel execution with ThreadStore propagation | +| `AutomationConstants` | String constants for all keys, statuses, and file names | diff --git a/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java index 160717aba73..0fe96a712f0 100644 --- a/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java @@ -29,6 +29,9 @@ import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import prerna.om.ThreadStore; import prerna.project.api.IProject; import prerna.reactor.automation.AutomationExecutionUtils; @@ -52,6 +55,8 @@ */ public final class AppEngineNodeExecutor implements IAutomationNodeExecutor { + private static final Logger classLogger = LogManager.getLogger(AppEngineNodeExecutor.class); + @Override public Object execute(AutomationNodeContext ctx) throws Exception { Map config = ctx.config(); @@ -64,6 +69,8 @@ public Object execute(AutomationNodeContext ctx) throws Exception { String resolvedPixel = AutomationExecutionUtils.resolve(pixel, scope, configMap); String resolvedAppId = appId != null ? AutomationExecutionUtils.resolve(appId, scope, configMap) : null; + classLogger.debug("App-engine node \"{}\" executing pixel in appId={}", nodeLabel, resolvedAppId != null ? resolvedAppId : "caller context"); + if (resolvedAppId != null && !resolvedAppId.isBlank()) { IProject project = Utility.getProject(resolvedAppId); if (project == null) { From 707d6172d7a5c2ada8537b9502d349dc8a94df8c Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Tue, 28 Jul 2026 16:58:16 -0400 Subject: [PATCH 10/25] feat: add async trigger pattern --- .../automation/AutomationDatabaseUtility.java | 27 ++++ .../automation/AutomationExecutionUtils.java | 74 ++--------- .../automation/AutomationRunEngine.java | 17 +-- .../GetActiveAutomationRunReactor.java | 101 ++++++++++++++ .../GetAutomationConfigReactor.java | 27 +++- .../automation/GetAutomationReactor.java | 9 +- src/prerna/reactor/automation/README.md | 22 ++-- .../automation/RunAutomationNodeReactor.java | 2 +- .../automation/TriggerAutomationReactor.java | 124 ++++++++---------- .../scheduler/SchedulerOwlCreator.java | 5 + 10 files changed, 255 insertions(+), 153 deletions(-) create mode 100644 src/prerna/reactor/automation/GetActiveAutomationRunReactor.java diff --git a/src/prerna/reactor/automation/AutomationDatabaseUtility.java b/src/prerna/reactor/automation/AutomationDatabaseUtility.java index f24e7a919a4..b191d70dc76 100644 --- a/src/prerna/reactor/automation/AutomationDatabaseUtility.java +++ b/src/prerna/reactor/automation/AutomationDatabaseUtility.java @@ -71,6 +71,7 @@ public final class AutomationDatabaseUtility { // Table name shortcuts for SelectQueryStruct (TABLE__COLUMN format) private static final String TABLE_RUNS = AutomationConstants.TABLE_AUTOMATION_RUNS; private static final String TABLE_NODE_OUTPUTS = AutomationConstants.TABLE_AUTOMATION_NODE_OUTPUTS; + private static final String TABLE_ACTIVE_RUN = AutomationConstants.TABLE_AUTOMATION_ACTIVE_RUN; private AutomationDatabaseUtility() { // static utility - no instantiation @@ -342,6 +343,32 @@ public static boolean releaseActiveRun(String projectId, String runId) { } } + /** + * Returns the active run ID for a project directly from the {@code AUTOMATION_ACTIVE_RUN} lock + * table. Unlike {@link #getActiveRun(String)}, this is populated at {@link #claimActiveRun} time + * — before {@code AUTOMATION_RUNS} is written — so callers polling for a newly started run will + * see it sooner. + * + * @return the run ID, or {@code null} if no run is currently active for the project + */ + public static String getClaimedActiveRun(String projectId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return null; + + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector(TABLE_ACTIVE_RUN + "__RUN_ID", "RUN_ID")); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_ACTIVE_RUN + "__PROJECT_ID", "==", projectId, PixelDataType.CONST_STRING)); + qs.setLimit(1); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + if (results != null && !results.isEmpty()) { + Object runId = results.get(0).get("RUN_ID"); + return runId != null ? runId.toString() : null; + } + return null; + } + /** * Sets the cluster-safe cancellation flag on a run. Called by {@code CancelAutomationRunReactor} * regardless of which pod receives the cancel request - unlike the in-memory diff --git a/src/prerna/reactor/automation/AutomationExecutionUtils.java b/src/prerna/reactor/automation/AutomationExecutionUtils.java index eba2784f601..5506864137d 100644 --- a/src/prerna/reactor/automation/AutomationExecutionUtils.java +++ b/src/prerna/reactor/automation/AutomationExecutionUtils.java @@ -31,13 +31,13 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; -import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.HashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Queue; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -46,6 +46,7 @@ import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; +import prerna.auth.User; import prerna.util.AssetUtility; /** @@ -286,13 +287,18 @@ private static Map parseJson(String json) { /** * Builds the initial variable scope for an automation run, seeded with {@code date}, * {@code triggered_at}, and {@code run_id} (when non-blank). + * + * @param runId the run ID to seed into scope, or {@code null} for test runs + * @param user the triggering user — used to localise {@code date} and {@code triggered_at} + * to the user's configured timezone; falls back to UTC when {@code null} or + * when no zone has been set on the user */ - public static Map buildInitialScope(String runId) { + public static Map buildInitialScope(String runId, User user) { Map scope = new HashMap<>(); - // TODO: use user's session timezone instead of UTC when the Insight timezone API is finalized - String now = Instant.now().toString(); - scope.put("date", now.substring(0, 10)); - scope.put("triggered_at", now); + ZoneId zone = (user != null && user.getZoneId() != null) ? user.getZoneId() : ZoneId.of("UTC"); + ZonedDateTime now = ZonedDateTime.now(zone); + scope.put("date", now.format(DateTimeFormatter.ISO_LOCAL_DATE)); + scope.put("triggered_at", now.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); if (runId != null && !runId.isBlank()) scope.put("run_id", runId); return scope; } @@ -352,56 +358,4 @@ public static Map loadAutomationDoc(String projectId) { } } - // -- Topological sort ---------------------------------------------------------- - - /** - * Topologically sorts a node/edge graph (Kahn's algorithm). Nodes with no incoming edges - * are seeded in node-array order, so a linear automation with no edges still runs - * top-to-bottom in the order the nodes were defined. - */ - @SuppressWarnings("unchecked") - public static List> topoSort(List> nodes, - List> edges) { - if (nodes == null || nodes.isEmpty()) return new ArrayList<>(); - - Map inDegree = new HashMap<>(); - Map> adj = new HashMap<>(); - - for (Map n : nodes) { - String id = (String) n.get("id"); - inDegree.put(id, 0); - adj.put(id, new ArrayList<>()); - } - if (edges != null) { - for (Map e : edges) { - String src = (String) e.get("source"); - String tgt = (String) e.get("target"); - adj.computeIfAbsent(src, k -> new ArrayList<>()).add(tgt); - inDegree.merge(tgt, 1, Integer::sum); - } - } - - Queue queue = new LinkedList<>(); - for (Map n : nodes) { - String id = (String) n.get("id"); - if (inDegree.getOrDefault(id, 0) == 0) queue.add(id); - } - - Map> nodeById = new HashMap<>(); - for (Map n : nodes) nodeById.put((String) n.get("id"), n); - - List> sorted = new ArrayList<>(); - while (!queue.isEmpty()) { - String id = queue.poll(); - sorted.add(nodeById.get(id)); - for (String neighbor : adj.getOrDefault(id, new ArrayList<>())) { - int deg = inDegree.merge(neighbor, -1, Integer::sum); - if (deg == 0) queue.add(neighbor); - } - } - if (sorted.size() != nodes.size()) { - throw new IllegalArgumentException("Automation graph contains a cycle — cannot determine execution order"); - } - return sorted; - } } diff --git a/src/prerna/reactor/automation/AutomationRunEngine.java b/src/prerna/reactor/automation/AutomationRunEngine.java index 874f388230f..32f47d85ba2 100644 --- a/src/prerna/reactor/automation/AutomationRunEngine.java +++ b/src/prerna/reactor/automation/AutomationRunEngine.java @@ -50,15 +50,16 @@ import prerna.util.Utility; /** - * Executes an automation run on a background thread. Separated from - * {@link TriggerAutomationReactor} so the reactor stays thin — it only parses params, - * claims a run slot, submits to the thread pool, and returns the run ID immediately. + * Executes an automation run synchronously. Called by {@link TriggerAutomationReactor} + * on the virtual thread provided by the platform's {@code runPixelAsync} endpoint. + * Iterates nodes in order, dispatches each to its {@link IAutomationNodeExecutor}, + * and writes per-node status to the DB as it goes. */ public final class AutomationRunEngine { private static final Logger classLogger = LogManager.getLogger(AutomationRunEngine.class); - /** In-memory cancellation flags — fast path; the DB flag is the cluster-safe source of truth. */ + /** In-memory cancellation flags - fast path; the DB flag is the cluster-safe source of truth. */ static final ConcurrentHashMap CANCELLATION_FLAGS = new ConcurrentHashMap<>(); private AutomationRunEngine() {} @@ -78,12 +79,12 @@ public static boolean requestCancellation(String runId) { /** * Runs the full automation node list, blocking until all nodes complete or the run is - * cancelled/failed. Must be called from a background thread (the automation executor). + * cancelled/failed. Runs synchronously on the calling virtual thread. * * @param runId the run record ID (already inserted into DB by the caller) * @param projectId the owning project - * @param ordered topologically sorted node list - * @param configMap project automation config key→value pairs + * @param ordered nodes in execution order + * @param configMap project automation config key-value pairs * @param insight the caller's insight context (propagated to each node executor) */ public static void run(String runId, String projectId, @@ -93,7 +94,7 @@ public static void run(String runId, String projectId, CANCELLATION_FLAGS.put(runId, cancelled); ScheduledExecutorService heartbeat = startHeartbeat(runId); - Map scope = AutomationExecutionUtils.buildInitialScope(runId); + Map scope = AutomationExecutionUtils.buildInitialScope(runId, insight.getUser()); int completedCount = 0; try { diff --git a/src/prerna/reactor/automation/GetActiveAutomationRunReactor.java b/src/prerna/reactor/automation/GetActiveAutomationRunReactor.java new file mode 100644 index 00000000000..cd85e12d0f7 --- /dev/null +++ b/src/prerna/reactor/automation/GetActiveAutomationRunReactor.java @@ -0,0 +1,101 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; + +/** + * Returns the currently active run ID for a project by reading the + * {@code AUTOMATION_ACTIVE_RUN} lock table directly. This table is populated by + * {@link AutomationDatabaseUtility#claimActiveRun} before + * {@code AUTOMATION_RUNS} is written, so the FE can discover the run ID while + * {@link TriggerAutomationReactor} is still executing synchronously on a virtual + * thread. + * + *

Returns {@code { RUN_ID, PROJECT_ID }} when a run is active, or an empty + * map when no run is in progress. + * + *

Pixel: {@code GetActiveAutomationRun(project=["appId"])} + */ +public class GetActiveAutomationRunReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GetActiveAutomationRunReactor.class); + + public GetActiveAutomationRunReactor() { + this.keysToGet = new String[] { "project" }; + this.keyRequired = new int[] { 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = getProjectId(); + + String runId = AutomationDatabaseUtility.getClaimedActiveRun(projectId); + + Map result = new HashMap<>(); + if (runId != null) { + result.put(AutomationConstants.RUN_ID, runId); + result.put(AutomationConstants.PROJECT_ID, projectId); + } + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + private String getProjectId() { + String projectId = this.keyValue.get(this.keysToGet[0]); + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + return projectId; + } + + @Override + public String getReactorDescription() { + return "Returns the active run ID from the AUTOMATION_ACTIVE_RUN table; empty map when no run is in progress."; + } + + @Override + protected String getDescriptionForKey(String key) { + if ("project".equals(key)) return "The project (app) ID or alias to check for an active run."; + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/reactor/automation/GetAutomationConfigReactor.java b/src/prerna/reactor/automation/GetAutomationConfigReactor.java index 0eb1444f0d2..4ce339b0fe4 100644 --- a/src/prerna/reactor/automation/GetAutomationConfigReactor.java +++ b/src/prerna/reactor/automation/GetAutomationConfigReactor.java @@ -38,8 +38,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; import prerna.auth.utils.SecurityProjectUtils; @@ -49,13 +47,25 @@ import prerna.sablecc2.om.nounmeta.NounMetadata; import prerna.util.AssetUtility; +/** + * Returns the automation environment config ({@code automation_config.json}) for a project. + * Sensitive values are masked in the response — only the key and a placeholder are returned. + * + *

There are three "get" reactors — each reads something different: + *

    + *
  • {@link GetAutomationReactor GetAutomation} — reads {@code automation.json}: the pipeline graph
    • + *
  • {@code GetAutomationConfig} (this reactor) — reads {@code automation_config.json}: key/value env vars and secrets
    • + *
  • {@link GetAutomationRunReactor GetAutomationRun} — reads live run state from the DB
    • + *
+ * + *

Pixel: {@code GetAutomationConfig(project=["appId"])} + */ public class GetAutomationConfigReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(GetAutomationConfigReactor.class); - private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); public GetAutomationConfigReactor() { - this.keysToGet = new String[]{ "project" }; + this.keysToGet = new String[] { "project" }; } @Override @@ -81,7 +91,7 @@ public NounMetadata execute() { try { String json = Files.readString(configFile.toPath(), StandardCharsets.UTF_8); - List> entries = GSON.fromJson(json, new TypeToken>>() {}.getType()); + List> entries = AutomationExecutionUtils.GSON.fromJson(json, new TypeToken>>() {}.getType()); if (entries != null) { for (Map entry : entries) { Object sensitive = entry.get("sensitive"); @@ -92,8 +102,13 @@ public NounMetadata execute() { } return new NounMetadata(entries != null ? entries : new ArrayList<>(), PixelDataType.VECTOR, PixelOperationType.OPERATION); } catch (IOException e) { - classLogger.error("Error reading automation config", e); + classLogger.error("Error reading automation_config.json for project {}", projectId, e); return new NounMetadata(new ArrayList<>(), PixelDataType.VECTOR, PixelOperationType.OPERATION); } } + + @Override + public String getReactorDescription() { + return "Returns the automation environment config (automation_config.json) for a project; sensitive values are masked."; + } } diff --git a/src/prerna/reactor/automation/GetAutomationReactor.java b/src/prerna/reactor/automation/GetAutomationReactor.java index c07ff49db41..7b6f52e95a3 100644 --- a/src/prerna/reactor/automation/GetAutomationReactor.java +++ b/src/prerna/reactor/automation/GetAutomationReactor.java @@ -88,7 +88,7 @@ public NounMetadata execute() { } IProject project = Utility.getProject(projectId); - if (project.requirePublish(true)) { + if (project != null && project.requirePublish(true)) { classLogger.info("Pulled project {} from cluster", projectId); } @@ -112,8 +112,13 @@ public NounMetadata execute() { new TypeToken>() {}.getType()); return new NounMetadata(doc, PixelDataType.MAP, PixelOperationType.OPERATION); } catch (IOException e) { - classLogger.error("Error reading automation JSON", e); + classLogger.error("Error reading automation.json for project {}", projectId, e); throw new IllegalArgumentException("Unable to read automation: " + e.getMessage()); } } + + @Override + public String getReactorDescription() { + return "Returns the automation pipeline definition (automation.json) for a project; returns an empty graph when none has been saved."; + } } diff --git a/src/prerna/reactor/automation/README.md b/src/prerna/reactor/automation/README.md index 8a23e0759fc..42b15e3420f 100644 --- a/src/prerna/reactor/automation/README.md +++ b/src/prerna/reactor/automation/README.md @@ -5,24 +5,30 @@ Executes sequential node pipelines against SEMOSS engines. Users build a pipelin ## How it works ``` -TriggerAutomationReactor +FE calls runPixelAsync("TriggerAutomation(...)") + → Monolith spawns a virtual thread, returns jobId immediately + → FE polls GetActiveAutomationRun every 500ms (up to 10×) to get runId + +TriggerAutomationReactor (virtual thread, synchronous) → reads automation.json (nodes in order) - → claims single-run slot (AUTOMATION_ACTIVE_RUN) - → submits to ThreadPoolExecutor (2–20 threads) - → returns runId immediately + → claims single-run slot (AUTOMATION_ACTIVE_RUN) → runId visible to FE + → inserts AUTOMATION_RUNS + AUTOMATION_NODE_OUTPUTS rows + → calls AutomationRunEngine.run() synchronously + → returns completed run result to jobId slot when done -AutomationRunEngine (background thread) +AutomationRunEngine (same virtual thread) → iterates nodes in saved order → dispatches each node to its IAutomationNodeExecutor - → writes node output + status to AUTOMATION_NODE_OUTPUTS - → FE polls GetAutomationRunReactor every 3s + → writes node output + status to AUTOMATION_NODE_OUTPUTS after each node + → FE polls GetAutomationRun every 3s while runId is known ``` ## Reactors | Reactor | Pixel | What it does | | --- | --- | --- | -| `TriggerAutomationReactor` | `TriggerAutomation(project=["id"])` | Starts a run, returns runId | +| `TriggerAutomationReactor` | `TriggerAutomation(project=["id"])` | Starts a run synchronously; returns completed run result | +| `GetActiveAutomationRunReactor` | `GetActiveAutomationRun(project=["id"])` | Returns `{RUN_ID, PROJECT_ID}` from active-run lock table; empty map when idle | | `GetAutomationReactor` | `GetAutomation(project=["id"])` | Returns saved pipeline definition (automation.json) | | `GetAutomationConfigReactor` | `GetAutomationConfig(project=["id"])` | Returns env var/secret config; masks sensitive values | | `GetAutomationRunReactor` | `GetAutomationRun(project=["id"], runId=["id"])` | Returns live run state for FE polling | diff --git a/src/prerna/reactor/automation/RunAutomationNodeReactor.java b/src/prerna/reactor/automation/RunAutomationNodeReactor.java index 61df2566069..d5dab53409c 100644 --- a/src/prerna/reactor/automation/RunAutomationNodeReactor.java +++ b/src/prerna/reactor/automation/RunAutomationNodeReactor.java @@ -145,7 +145,7 @@ private static Map findNode(String projectId, String nodeId) { } private Map buildScope(String contextRunId) { - Map scope = AutomationExecutionUtils.buildInitialScope(null); + Map scope = AutomationExecutionUtils.buildInitialScope(null, this.insight.getUser()); if (contextRunId != null && !contextRunId.isEmpty()) { List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(contextRunId); diff --git a/src/prerna/reactor/automation/TriggerAutomationReactor.java b/src/prerna/reactor/automation/TriggerAutomationReactor.java index b1e825adefb..f71388d6362 100644 --- a/src/prerna/reactor/automation/TriggerAutomationReactor.java +++ b/src/prerna/reactor/automation/TriggerAutomationReactor.java @@ -32,25 +32,23 @@ import java.util.List; import java.util.Map; import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import prerna.auth.utils.SecurityProjectUtils; -import prerna.om.ThreadStore; import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.reactor.agent.mcp.MCPUtility.MCPDisplayOption; +import prerna.reactor.agent.mcp.MCPUtility.MCPExecution; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; import prerna.sablecc2.om.nounmeta.NounMetadata; /** * Manually triggers an automation run for a project. Validates access, claims the single-run slot, - * submits execution to a background thread pool, and returns the run ID immediately for polling. + * runs the automation synchronously on the calling thread (expected to be a virtual thread from + * the platform's {@code runPixelAsync} endpoint), and returns the completed run result. * *

Pixel: {@code TriggerAutomation(project=["appId"])} */ @@ -58,17 +56,6 @@ public class TriggerAutomationReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(TriggerAutomationReactor.class); - private static final ExecutorService AUTOMATION_EXECUTOR = new ThreadPoolExecutor( - 2, 20, 60L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>(10), - r -> { - Thread t = new Thread(r, "automation-run-" + System.nanoTime()); - t.setDaemon(true); - return t; - }, - new ThreadPoolExecutor.AbortPolicy() - ); - public TriggerAutomationReactor() { this.keysToGet = new String[] { "project" }; this.keyRequired = new int[] { 1 }; @@ -88,19 +75,16 @@ public NounMetadata execute() { ". Wait for it to complete or cancel it before starting a new run."); } + boolean runStarted = false; try { Map doc = AutomationExecutionUtils.loadAutomationDoc(projectId); @SuppressWarnings("unchecked") Map graph = (Map) doc.get("graph"); @SuppressWarnings("unchecked") List> nodes = (List>) graph.get("nodes"); - @SuppressWarnings("unchecked") - List> edges = (List>) graph.get("edges"); Map configMap = AutomationExecutionUtils.loadConfig(projectId); - // The form view always produces a sequential node list with no edges, so topoSort - // degrades to node-list order. Edges + sort are kept for a future visual flow editor. - List> ordered = AutomationExecutionUtils.topoSort(nodes, edges); + List> ordered = nodes != null ? nodes : new ArrayList<>(); if (ordered.isEmpty()) { throw new IllegalArgumentException("Automation has no nodes to execute"); } @@ -109,58 +93,48 @@ public NounMetadata execute() { AutomationConstants.TRIGGER_MANUAL, ordered.size(), userId); AutomationDatabaseUtility.insertAllNodeOutputs(runId, ordered); - Map parentContext = ThreadStore.getTheadMapObject(); - final Map contextSnapshot = - parentContext != null ? new HashMap<>(parentContext) : null; - - try { - AUTOMATION_EXECUTOR.submit(() -> { - installThreadContext(contextSnapshot); - try { - AutomationRunEngine.run(runId, projectId, ordered, configMap, this.insight); - } catch (Exception e) { - classLogger.error("Unhandled error in automation run {}: {}", runId, e.getMessage(), e); - AutomationDatabaseUtility.updateRunStatus(runId, - AutomationConstants.STATUS_FAILED, null, e.getMessage()); - } finally { - ThreadStore.remove(); - } - }); - } catch (RejectedExecutionException e) { - AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_FAILED, - null, "Server is at capacity - too many concurrent automation runs"); - throw new IllegalStateException("Too many concurrent automation runs. Please try again shortly."); + classLogger.info("Automation run {} starting for project {}", runId, projectId); + runStarted = true; + AutomationRunEngine.run(runId, projectId, ordered, configMap, this.insight); + + // Build final result in the same shape as GetAutomationRunReactor + Map runDetail = AutomationDatabaseUtility.getRunDetail(runId); + List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(runId); + List> nodeResults = new ArrayList<>(); + if (nodeOutputs != null) { + for (Map output : nodeOutputs) { + Map nodeResult = new HashMap<>(); + nodeResult.put(AutomationConstants.NODE_ID, output.get(AutomationConstants.NODE_ID)); + nodeResult.put(AutomationConstants.NODE_LABEL, output.get(AutomationConstants.NODE_LABEL)); + nodeResult.put(AutomationConstants.STATUS, output.get(AutomationConstants.STATUS)); + nodeResult.put(AutomationConstants.DURATION_MS, output.get(AutomationConstants.DURATION_MS)); + nodeResult.put(AutomationConstants.OUTPUT_PREVIEW, output.get(AutomationConstants.OUTPUT_PREVIEW)); + nodeResult.put(AutomationConstants.ERROR_MESSAGE, output.get(AutomationConstants.ERROR_MESSAGE)); + nodeResults.add(nodeResult); + } } - - classLogger.info("Automation run {} submitted for project {}", runId, projectId); - - Map stored = AutomationDatabaseUtility.getRunDetail(runId); - Map result = new HashMap<>(); - result.put(AutomationConstants.RUN_ID, runId); - result.put(AutomationConstants.PROJECT_ID, projectId); - result.put(AutomationConstants.STATUS, AutomationConstants.STATUS_RUNNING); - result.put(AutomationConstants.TOTAL_NODES, ordered.size()); - result.put(AutomationConstants.COMPLETED_NODES, 0); - if (stored != null) { - result.put(AutomationConstants.STARTED_AT, stored.get(AutomationConstants.STARTED_AT)); + if (runDetail == null) { + runDetail = new HashMap<>(); + runDetail.put(AutomationConstants.RUN_ID, runId); + runDetail.put(AutomationConstants.PROJECT_ID, projectId); } - result.put("nodeResults", new ArrayList<>()); - return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); - - } catch (RuntimeException e) { - AutomationDatabaseUtility.releaseActiveRun(projectId, runId); - throw e; + runDetail.put("nodeResults", nodeResults); + return new NounMetadata(runDetail, PixelDataType.MAP, PixelOperationType.OPERATION); + + } catch (Exception e) { + classLogger.error("Automation run setup failed for project {}, run {}", projectId, runId, e); + // Only release if AutomationRunEngine.run() was never called - once it starts, + // its own finally block handles releaseActiveRun. + if (!runStarted) { + AutomationDatabaseUtility.releaseActiveRun(projectId, runId); + } + if (e instanceof RuntimeException re) throw re; + throw new RuntimeException(e); } } // -- Helpers ------------------------------------------------------------------- - private static void installThreadContext(Map snapshot) { - if (snapshot == null || snapshot.isEmpty()) return; - ThreadStore.getInsightId(); - ThreadStore.setThreadMapObject(snapshot); - } - private String getProjectId() { String projectId = this.keyValue.get(this.keysToGet[0]); if (projectId == null || projectId.isEmpty()) { @@ -180,8 +154,22 @@ private String getUserId() { return "system"; } + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPExecution.ASK.getValue()); + meta.put(MCPUtility.UI_DISPLAY_LOCATION, MCPDisplayOption.SIDEBAR.getValue()); + return meta; + } + @Override public String getReactorDescription() { return "Manually triggers an automation run for the given project and returns a run ID for polling."; } + + @Override + protected String getDescriptionForKey(String key) { + if ("project".equals(key)) return "The project (app) ID or alias to run the automation for."; + return super.getDescriptionForKey(key); + } } diff --git a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java index 44560410023..246f125b82b 100644 --- a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java +++ b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java @@ -307,6 +307,11 @@ public void createColumnsAndTypes() { Pair.with(AutomationConstants.OUTPUT_VALUE, CLOB), Pair.with(AutomationConstants.OUTPUT_PREVIEW, VARCHAR_2000), Pair.with(AutomationConstants.ERROR_MESSAGE, CLOB))); + + addTable(AutomationConstants.TABLE_AUTOMATION_ACTIVE_RUN, Arrays.asList( + Pair.with(AutomationConstants.PROJECT_ID, VARCHAR_255), + Pair.with(AutomationConstants.RUN_ID, VARCHAR_255), + Pair.with(AutomationConstants.CLAIMED_AT, TIMESTAMP))); // @formatter:on } From c8a2d1a231408fd7ad287456f3aafb580a9eefe3 Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Wed, 29 Jul 2026 10:23:23 -0400 Subject: [PATCH 11/25] chore: code clean up --- .../automation/AutomationConstants.java | 116 ++++++++ .../automation/AutomationDatabaseUtility.java | 257 +++++++++++------- .../automation/AutomationExecutionUtils.java | 66 +++-- .../automation/AutomationRunEngine.java | 24 +- .../CancelAutomationRunReactor.java | 31 ++- .../GetActiveAutomationRunReactor.java | 5 +- .../GetAutomationConfigReactor.java | 7 +- .../automation/GetAutomationReactor.java | 11 +- .../automation/GetAutomationRunReactor.java | 20 +- .../automation/ListAutomationRunsReactor.java | 12 +- .../automation/RunAutomationNodeReactor.java | 44 ++- .../SaveAutomationConfigReactor.java | 40 ++- .../automation/SaveAutomationReactor.java | 20 +- .../automation/TriggerAutomationReactor.java | 13 +- .../nodes/AppEngineNodeExecutor.java | 15 +- .../nodes/AutomationNodeContext.java | 11 +- .../nodes/DatabaseEngineNodeExecutor.java | 18 +- .../nodes/FunctionEngineNodeExecutor.java | 13 +- .../nodes/ModelEngineNodeExecutor.java | 23 +- .../nodes/StorageEngineNodeExecutor.java | 40 ++- .../nodes/VectorEngineNodeExecutor.java | 35 ++- .../automation/nodes/WaitNodeExecutor.java | 22 +- 22 files changed, 592 insertions(+), 251 deletions(-) diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index 15409f3d392..b49cb4a1a71 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -106,18 +106,134 @@ private AutomationConstants() {} public static final String NODE_WAIT = "wait"; public static final String NODE_APP = "app"; + // -- Node config keys (node.config map fields, shared across executors) -------- + + public static final String CONFIG_ENGINE_ID = "engineId"; + public static final String CONFIG_OPERATION = "operation"; + public static final String CONFIG_EXPRESSION = "expression"; + public static final String CONFIG_LIMIT = "limit"; + public static final String CONFIG_VALUES = "values"; + public static final String CONFIG_COMMAND = "command"; + public static final String CONFIG_CONTEXT = "context"; + public static final String CONFIG_PARAM_VALUES = "paramValues"; + public static final String CONFIG_PARAMS = "params"; + public static final String CONFIG_STORAGE_PATH = "storagePath"; + public static final String CONFIG_FILE_PATH = "filePath"; + public static final String CONFIG_FILE_NAMES = "fileNames"; + public static final String CONFIG_SECONDS = "seconds"; + public static final String CONFIG_PIXEL = "pixel"; + public static final String CONFIG_APP_ID = "appId"; + public static final String CONFIG_TIMEOUT_SECONDS = "timeoutSeconds"; + public static final String DEFAULT_STORAGE_PATH = "/"; + public static final String EMPTY_JSON_OBJECT = "{}"; + public static final String EMPTY_JSON_ARRAY = "[]"; + + // -- Node operation values ------------------------------------------------------- + + public static final String OP_READ = "read"; + public static final String OP_WRITE = "write"; + public static final String OP_LLM = "llm"; + public static final String OP_EMBEDDINGS = "embeddings"; + public static final String OP_VISION = "vision"; + public static final String OP_NER = "ner"; + public static final String OP_SEARCH = "search"; + public static final String OP_ADD_FILE = "add-file"; + public static final String OP_ADD_CSV = "add-csv"; + public static final String OP_LIST = "list"; + public static final String OP_DELETE = "delete"; + public static final String OP_DOWNLOAD = "download"; + public static final String OP_UPLOAD = "upload"; + public static final String OP_READ_BASE64 = "read-base64"; + + // -- Node execution defaults / bounds -------------------------------------------- + + public static final int DEFAULT_DB_QUERY_LIMIT = 50; + public static final int DEFAULT_VECTOR_SEARCH_LIMIT = 5; + public static final int DEFAULT_LIST_RUNS_LIMIT = 25; + public static final int WAIT_MIN_SECONDS = 0; + public static final int WAIT_MAX_SECONDS = 3600; + public static final int WAIT_DEFAULT_SECONDS = 1; + public static final int WAIT_CANCEL_CHECK_INTERVAL_SECONDS = 5; + + // -- automation.json document field names ---------------------------------------- + + public static final String DOC_VERSION = "version"; + public static final String DOC_GRAPH = "graph"; + public static final String DOC_NODES = "nodes"; + public static final String DOC_EDGES = "edges"; + public static final int DOC_CURRENT_VERSION = 1; + + // -- Node/edge field names -------------------------------------------------------- + + public static final String NODE_FIELD_ID = "id"; + public static final String NODE_FIELD_TYPE = "type"; + public static final String NODE_FIELD_LABEL = "label"; + public static final String NODE_FIELD_CONFIG = "config"; + public static final String NODE_FIELD_OUTPUT_TRANSFORM = "outputTransform"; + public static final String NODE_FIELD_OUTPUT_VAR = "outputVar"; + public static final String EDGE_FIELD_SOURCE = "source"; + public static final String EDGE_FIELD_TARGET = "target"; + public static final String UNNAMED_NODE_LABEL = "unnamed"; + + // -- automation-config.json entry field names ------------------------------------ + + public static final String CONFIG_ENTRY_KEY = "key"; + public static final String CONFIG_ENTRY_VALUE = "value"; + public static final String CONFIG_ENTRY_SENSITIVE = "sensitive"; + + // -- Output transform field names / modes ---------------------------------------- + + public static final String TRANSFORM_MODE = "mode"; + public static final String TRANSFORM_COLUMN = "column"; + public static final String TRANSFORM_PATH = "path"; + public static final String TRANSFORM_MODE_RAW = "raw"; + public static final String TRANSFORM_MODE_ROWS_AS_OBJECTS = "rows-as-objects"; + public static final String TRANSFORM_MODE_FIRST_ROW = "first-row"; + public static final String TRANSFORM_MODE_COLUMN = "column"; + public static final String TRANSFORM_MODE_JSONPATH = "jsonpath"; + public static final String DATASET_HEADERS = "headers"; + public static final String DATASET_VALUES = "values"; + public static final String DATASET_DATA = "data"; + + // -- Scope variable names --------------------------------------------------------- + + public static final String SCOPE_DATE = "date"; + public static final String SCOPE_TRIGGERED_AT = "triggered_at"; + public static final String SCOPE_RUN_ID = "run_id"; + public static final String TEST_RUN_ID = "test"; + public static final String SYSTEM_USER_ID = "system"; + + // -- Result map keys --------------------------------------------------------------- + + public static final String RESULT_NODE_RESULTS = "nodeResults"; + public static final String RESULT_CANCEL_REQUESTED = "cancelRequested"; + public static final String RESULT_SIGNALLED_LOCALLY = "signalledLocally"; + public static final String RESULT_OUTPUT_VALUE = "outputValue"; + // -- Pixel execution defaults ---------------------------------------------------- public static final int DEFAULT_TIMEOUT_SECONDS = 300; // -- Data type constants (for table creation) ---------------------------------- + public static final String VARCHAR_50 = "VARCHAR(50)"; public static final String VARCHAR_255 = "VARCHAR(255)"; public static final String VARCHAR_500 = "VARCHAR(500)"; + public static final String VARCHAR_2000 = "VARCHAR(2000)"; public static final String INTEGER = "INTEGER"; public static final String BIGINT = "BIGINT"; public static final String NOT_NULL = "NOT NULL"; + // -- DDL object names (indexes / primary keys) ---------------------------------- + + public static final String PK_AUTOMATION_RUNS = "PK_AUTOMATION_RUNS"; + public static final String PK_AUTO_NODE_OUT = "PK_AUTO_NODE_OUT"; + public static final String PK_AUTO_ACTIVE_RUN = "PK_AUTO_ACTIVE_RUN"; + public static final String IDX_AR_PROJECT = "IDX_AR_PROJECT"; + public static final String IDX_AR_STATUS = "IDX_AR_STATUS"; + public static final String IDX_AR_STARTED = "IDX_AR_STARTED"; + public static final String IDX_ANO_RUN = "IDX_ANO_RUN"; + // -- Defaults ------------------------------------------------------------------ public static final String DEFAULT_AUTOMATION_ID = "default"; diff --git a/src/prerna/reactor/automation/AutomationDatabaseUtility.java b/src/prerna/reactor/automation/AutomationDatabaseUtility.java index b191d70dc76..ec6e82e5fbe 100644 --- a/src/prerna/reactor/automation/AutomationDatabaseUtility.java +++ b/src/prerna/reactor/automation/AutomationDatabaseUtility.java @@ -27,6 +27,55 @@ *******************************************************************************/ package prerna.reactor.automation; +import static prerna.reactor.automation.AutomationConstants.AUTOMATION_ID; +import static prerna.reactor.automation.AutomationConstants.BIGINT; +import static prerna.reactor.automation.AutomationConstants.CANCEL_REQUESTED; +import static prerna.reactor.automation.AutomationConstants.CLAIMED_AT; +import static prerna.reactor.automation.AutomationConstants.COMPLETED_AT; +import static prerna.reactor.automation.AutomationConstants.COMPLETED_NODES; +import static prerna.reactor.automation.AutomationConstants.CREATED_BY; +import static prerna.reactor.automation.AutomationConstants.DURATION_MS; +import static prerna.reactor.automation.AutomationConstants.ERROR_MESSAGE; +import static prerna.reactor.automation.AutomationConstants.EXECUTION_ORDER; +import static prerna.reactor.automation.AutomationConstants.FAILED_NODE_ID; +import static prerna.reactor.automation.AutomationConstants.IDX_ANO_RUN; +import static prerna.reactor.automation.AutomationConstants.IDX_AR_PROJECT; +import static prerna.reactor.automation.AutomationConstants.IDX_AR_STARTED; +import static prerna.reactor.automation.AutomationConstants.IDX_AR_STATUS; +import static prerna.reactor.automation.AutomationConstants.INTEGER; +import static prerna.reactor.automation.AutomationConstants.LAST_HEARTBEAT; +import static prerna.reactor.automation.AutomationConstants.NODE_FIELD_ID; +import static prerna.reactor.automation.AutomationConstants.NODE_FIELD_LABEL; +import static prerna.reactor.automation.AutomationConstants.NODE_ID; +import static prerna.reactor.automation.AutomationConstants.NODE_LABEL; +import static prerna.reactor.automation.AutomationConstants.NODE_STATUS_FAILED; +import static prerna.reactor.automation.AutomationConstants.NODE_STATUS_PENDING; +import static prerna.reactor.automation.AutomationConstants.NODE_STATUS_RUNNING; +import static prerna.reactor.automation.AutomationConstants.NODE_STATUS_SUCCESS; +import static prerna.reactor.automation.AutomationConstants.NOT_NULL; +import static prerna.reactor.automation.AutomationConstants.OUTPUT_PREVIEW; +import static prerna.reactor.automation.AutomationConstants.OUTPUT_VALUE; +import static prerna.reactor.automation.AutomationConstants.OUTPUT_VAR; +import static prerna.reactor.automation.AutomationConstants.PK_AUTOMATION_RUNS; +import static prerna.reactor.automation.AutomationConstants.PK_AUTO_ACTIVE_RUN; +import static prerna.reactor.automation.AutomationConstants.PK_AUTO_NODE_OUT; +import static prerna.reactor.automation.AutomationConstants.PROJECT_ID; +import static prerna.reactor.automation.AutomationConstants.RUN_ID; +import static prerna.reactor.automation.AutomationConstants.STALE_HEARTBEAT_THRESHOLD_MINUTES; +import static prerna.reactor.automation.AutomationConstants.STARTED_AT; +import static prerna.reactor.automation.AutomationConstants.STATUS; +import static prerna.reactor.automation.AutomationConstants.STATUS_INTERRUPTED; +import static prerna.reactor.automation.AutomationConstants.STATUS_RUNNING; +import static prerna.reactor.automation.AutomationConstants.TABLE_AUTOMATION_ACTIVE_RUN; +import static prerna.reactor.automation.AutomationConstants.TABLE_AUTOMATION_NODE_OUTPUTS; +import static prerna.reactor.automation.AutomationConstants.TABLE_AUTOMATION_RUNS; +import static prerna.reactor.automation.AutomationConstants.TOTAL_NODES; +import static prerna.reactor.automation.AutomationConstants.TRIGGER_TYPE; +import static prerna.reactor.automation.AutomationConstants.VARCHAR_2000; +import static prerna.reactor.automation.AutomationConstants.VARCHAR_255; +import static prerna.reactor.automation.AutomationConstants.VARCHAR_50; +import static prerna.reactor.automation.AutomationConstants.VARCHAR_500; + import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -69,9 +118,9 @@ public final class AutomationDatabaseUtility { private static final Logger classLogger = LogManager.getLogger(AutomationDatabaseUtility.class); // Table name shortcuts for SelectQueryStruct (TABLE__COLUMN format) - private static final String TABLE_RUNS = AutomationConstants.TABLE_AUTOMATION_RUNS; - private static final String TABLE_NODE_OUTPUTS = AutomationConstants.TABLE_AUTOMATION_NODE_OUTPUTS; - private static final String TABLE_ACTIVE_RUN = AutomationConstants.TABLE_AUTOMATION_ACTIVE_RUN; + private static final String TABLE_RUNS = TABLE_AUTOMATION_RUNS; + private static final String TABLE_NODE_OUTPUTS = TABLE_AUTOMATION_NODE_OUTPUTS; + private static final String TABLE_ACTIVE_RUN = TABLE_AUTOMATION_ACTIVE_RUN; private AutomationDatabaseUtility() { // static utility - no instantiation @@ -178,11 +227,11 @@ public static void markStaleRunsInterrupted() { if (schedulerDb == null) return; SelectQueryStruct qs = new SelectQueryStruct(); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__RUN_ID", "RUN_ID")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__PROJECT_ID", "PROJECT_ID")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__LAST_HEARTBEAT", "LAST_HEARTBEAT")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + RUN_ID, RUN_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + PROJECT_ID, PROJECT_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + LAST_HEARTBEAT, LAST_HEARTBEAT)); qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( - TABLE_RUNS + "__STATUS", "==", AutomationConstants.STATUS_RUNNING, PixelDataType.CONST_STRING)); + TABLE_RUNS + "__" + STATUS, "==", STATUS_RUNNING, PixelDataType.CONST_STRING)); List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); if (results == null || results.isEmpty()) { @@ -190,21 +239,21 @@ public static void markStaleRunsInterrupted() { } Timestamp threshold = toTimestamp(Instant.now().minusSeconds( - AutomationConstants.STALE_HEARTBEAT_THRESHOLD_MINUTES * 60L)); + STALE_HEARTBEAT_THRESHOLD_MINUTES * 60L)); Timestamp now = toTimestamp(Instant.now()); Connection conn = null; try { conn = schedulerDb.getConnection(); for (Map row : results) { - String runId = (String) row.get("RUN_ID"); - String projectId = (String) row.get("PROJECT_ID"); + String runId = (String) row.get(RUN_ID); + String projectId = (String) row.get(PROJECT_ID); // Only interrupt runs whose heartbeat is actually stale. A run with a fresh // heartbeat is still alive (e.g. executing on another node in a cluster), so // interrupting it would clobber active work. A missing/unparseable heartbeat // is treated as stale (a crashed run that never checkpointed). - Timestamp lastHeartbeat = toTimestampSafe(row.get("LAST_HEARTBEAT")); + Timestamp lastHeartbeat = toTimestampSafe(row.get(LAST_HEARTBEAT)); if (lastHeartbeat != null && lastHeartbeat.after(threshold)) { classLogger.debug("Skipping automation run {} - heartbeat {} is newer than stale threshold {}", runId, lastHeartbeat, threshold); @@ -213,7 +262,7 @@ public static void markStaleRunsInterrupted() { try (PreparedStatement ps = conn.prepareStatement(MARK_STALE_INTERRUPTED)) { int index = 1; - ps.setString(index++, AutomationConstants.STATUS_INTERRUPTED); + ps.setString(index++, STATUS_INTERRUPTED); ps.setTimestamp(index++, now); ps.setString(index++, "Server restarted during execution"); ps.setString(index++, runId); @@ -250,16 +299,16 @@ public static String getActiveRun(String projectId) { if (schedulerDb == null) return null; SelectQueryStruct qs = new SelectQueryStruct(); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__RUN_ID", "RUN_ID")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + RUN_ID, RUN_ID)); qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( - TABLE_RUNS + "__PROJECT_ID", "==", projectId, PixelDataType.CONST_STRING)); + TABLE_RUNS + "__" + PROJECT_ID, "==", projectId, PixelDataType.CONST_STRING)); qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( - TABLE_RUNS + "__STATUS", "==", AutomationConstants.STATUS_RUNNING, PixelDataType.CONST_STRING)); + TABLE_RUNS + "__" + STATUS, "==", STATUS_RUNNING, PixelDataType.CONST_STRING)); qs.setLimit(1); List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); if (results != null && !results.isEmpty()) { - Object runId = results.get(0).get("RUN_ID"); + Object runId = results.get(0).get(RUN_ID); return runId != null ? runId.toString() : null; } return null; @@ -356,14 +405,14 @@ public static String getClaimedActiveRun(String projectId) { if (schedulerDb == null) return null; SelectQueryStruct qs = new SelectQueryStruct(); - qs.addSelector(new QueryColumnSelector(TABLE_ACTIVE_RUN + "__RUN_ID", "RUN_ID")); + qs.addSelector(new QueryColumnSelector(TABLE_ACTIVE_RUN + "__" + RUN_ID, RUN_ID)); qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( - TABLE_ACTIVE_RUN + "__PROJECT_ID", "==", projectId, PixelDataType.CONST_STRING)); + TABLE_ACTIVE_RUN + "__" + PROJECT_ID, "==", projectId, PixelDataType.CONST_STRING)); qs.setLimit(1); List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); if (results != null && !results.isEmpty()) { - Object runId = results.get(0).get("RUN_ID"); + Object runId = results.get(0).get(RUN_ID); return runId != null ? runId.toString() : null; } return null; @@ -409,17 +458,17 @@ public static boolean isCancelRequested(String runId) { if (schedulerDb == null) return false; SelectQueryStruct qs = new SelectQueryStruct(); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + AutomationConstants.CANCEL_REQUESTED, - AutomationConstants.CANCEL_REQUESTED)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + CANCEL_REQUESTED, + CANCEL_REQUESTED)); qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( - TABLE_RUNS + "__RUN_ID", "==", runId, PixelDataType.CONST_STRING)); + TABLE_RUNS + "__" + RUN_ID, "==", runId, PixelDataType.CONST_STRING)); qs.setLimit(1); List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); if (results == null || results.isEmpty()) { return false; } - Object flag = results.get(0).get(AutomationConstants.CANCEL_REQUESTED); + Object flag = results.get(0).get(CANCEL_REQUESTED); if (flag instanceof Boolean) { return (Boolean) flag; } @@ -444,7 +493,7 @@ public static boolean insertRun(String runId, String projectId, String automatio ps.setString(index++, runId); ps.setString(index++, projectId); ps.setString(index++, automationId); - ps.setString(index++, AutomationConstants.STATUS_RUNNING); + ps.setString(index++, STATUS_RUNNING); ps.setString(index++, triggerType); ps.setTimestamp(index++, now); ps.setTimestamp(index++, now); @@ -566,21 +615,21 @@ public static List> getRunsForProject(String projectId, int if (schedulerDb == null) return new ArrayList<>(); SelectQueryStruct qs = new SelectQueryStruct(); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__RUN_ID", "RUN_ID")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__PROJECT_ID", "PROJECT_ID")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__AUTOMATION_ID", "AUTOMATION_ID")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__STATUS", "STATUS")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__TRIGGER_TYPE", "TRIGGER_TYPE")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__STARTED_AT", "STARTED_AT")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__COMPLETED_AT", "COMPLETED_AT")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__FAILED_NODE_ID", "FAILED_NODE_ID")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__TOTAL_NODES", "TOTAL_NODES")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__COMPLETED_NODES", "COMPLETED_NODES")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__CREATED_BY", "CREATED_BY")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + RUN_ID, RUN_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + PROJECT_ID, PROJECT_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + AUTOMATION_ID, AUTOMATION_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + STATUS, STATUS)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + TRIGGER_TYPE, TRIGGER_TYPE)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + STARTED_AT, STARTED_AT)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + COMPLETED_AT, COMPLETED_AT)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + FAILED_NODE_ID, FAILED_NODE_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + TOTAL_NODES, TOTAL_NODES)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + COMPLETED_NODES, COMPLETED_NODES)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + CREATED_BY, CREATED_BY)); qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( - TABLE_RUNS + "__PROJECT_ID", "==", projectId, PixelDataType.CONST_STRING)); - qs.addOrderBy(TABLE_RUNS + "__STARTED_AT", + TABLE_RUNS + "__" + PROJECT_ID, "==", projectId, PixelDataType.CONST_STRING)); + qs.addOrderBy(TABLE_RUNS + "__" + STARTED_AT, QueryColumnOrderBySelector.ORDER_BY_DIRECTION.DESC.toString()); qs.setLimit(limit); @@ -596,21 +645,21 @@ public static Map getRunDetail(String runId) { if (schedulerDb == null) return null; SelectQueryStruct qs = new SelectQueryStruct(); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__RUN_ID", "RUN_ID")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__PROJECT_ID", "PROJECT_ID")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__AUTOMATION_ID", "AUTOMATION_ID")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__STATUS", "STATUS")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__TRIGGER_TYPE", "TRIGGER_TYPE")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__STARTED_AT", "STARTED_AT")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__COMPLETED_AT", "COMPLETED_AT")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__FAILED_NODE_ID", "FAILED_NODE_ID")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__ERROR_MESSAGE", "ERROR_MESSAGE")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__TOTAL_NODES", "TOTAL_NODES")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__COMPLETED_NODES", "COMPLETED_NODES")); - qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__CREATED_BY", "CREATED_BY")); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + RUN_ID, RUN_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + PROJECT_ID, PROJECT_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + AUTOMATION_ID, AUTOMATION_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + STATUS, STATUS)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + TRIGGER_TYPE, TRIGGER_TYPE)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + STARTED_AT, STARTED_AT)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + COMPLETED_AT, COMPLETED_AT)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + FAILED_NODE_ID, FAILED_NODE_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + ERROR_MESSAGE, ERROR_MESSAGE)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + TOTAL_NODES, TOTAL_NODES)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + COMPLETED_NODES, COMPLETED_NODES)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + CREATED_BY, CREATED_BY)); qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( - TABLE_RUNS + "__RUN_ID", "==", runId, PixelDataType.CONST_STRING)); + TABLE_RUNS + "__" + RUN_ID, "==", runId, PixelDataType.CONST_STRING)); qs.setLimit(1); List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); @@ -637,10 +686,10 @@ public static boolean insertAllNodeOutputs(String runId, List node = orderedNodes.get(i); int index = 1; ps.setString(index++, runId); - ps.setString(index++, (String) node.get("id")); - ps.setString(index++, (String) node.get("label")); + ps.setString(index++, (String) node.get(NODE_FIELD_ID)); + ps.setString(index++, (String) node.get(NODE_FIELD_LABEL)); ps.setInt(index++, i); - ps.setString(index++, AutomationConstants.NODE_STATUS_PENDING); + ps.setString(index++, NODE_STATUS_PENDING); ps.addBatch(); } ps.executeBatch(); @@ -669,7 +718,7 @@ public static boolean markNodeRunning(String runId, String nodeId) { conn = schedulerDb.getConnection(); try (PreparedStatement ps = conn.prepareStatement(UPDATE_NODE_STATUS)) { int index = 1; - ps.setString(index++, AutomationConstants.NODE_STATUS_RUNNING); + ps.setString(index++, NODE_STATUS_RUNNING); ps.setTimestamp(index++, toTimestamp(Instant.now())); ps.setString(index++, runId); ps.setString(index++, nodeId); @@ -703,7 +752,7 @@ public static boolean updateNodeSuccess(String runId, String nodeId, Timestamp s try (PreparedStatement ps = conn.prepareStatement(UPDATE_NODE_OUTPUT_SUCCESS)) { int index = 1; - ps.setString(index++, AutomationConstants.NODE_STATUS_SUCCESS); + ps.setString(index++, NODE_STATUS_SUCCESS); ps.setTimestamp(index++, startedAt); ps.setTimestamp(index++, toTimestamp(Instant.now())); ps.setLong(index++, durationMs); @@ -741,7 +790,7 @@ public static boolean updateNodeFailed(String runId, String nodeId, Timestamp st conn = schedulerDb.getConnection(); try (PreparedStatement ps = conn.prepareStatement(UPDATE_NODE_OUTPUT_FAILED)) { int index = 1; - ps.setString(index++, AutomationConstants.NODE_STATUS_FAILED); + ps.setString(index++, NODE_STATUS_FAILED); ps.setTimestamp(index++, startedAt); ps.setTimestamp(index++, toTimestamp(Instant.now())); ps.setLong(index++, durationMs); @@ -771,21 +820,21 @@ public static List> getNodeOutputsForRun(String runId) { if (schedulerDb == null) return new ArrayList<>(); SelectQueryStruct qs = new SelectQueryStruct(); - qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__NODE_ID", "NODE_ID")); - qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__NODE_LABEL", "NODE_LABEL")); - qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__EXECUTION_ORDER", "EXECUTION_ORDER")); - qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__STATUS", "STATUS")); - qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__STARTED_AT", "STARTED_AT")); - qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__COMPLETED_AT", "COMPLETED_AT")); - qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__DURATION_MS", "DURATION_MS")); - qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__OUTPUT_VAR", "OUTPUT_VAR")); - qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__OUTPUT_VALUE", "OUTPUT_VALUE")); - qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__OUTPUT_PREVIEW", "OUTPUT_PREVIEW")); - qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__ERROR_MESSAGE", "ERROR_MESSAGE")); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + NODE_ID, NODE_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + NODE_LABEL, NODE_LABEL)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + EXECUTION_ORDER, EXECUTION_ORDER)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + STATUS, STATUS)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + STARTED_AT, STARTED_AT)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + COMPLETED_AT, COMPLETED_AT)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + DURATION_MS, DURATION_MS)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + OUTPUT_VAR, OUTPUT_VAR)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + OUTPUT_VALUE, OUTPUT_VALUE)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + OUTPUT_PREVIEW, OUTPUT_PREVIEW)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + ERROR_MESSAGE, ERROR_MESSAGE)); qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( - TABLE_NODE_OUTPUTS + "__RUN_ID", "==", runId, PixelDataType.CONST_STRING)); - qs.addOrderBy(TABLE_NODE_OUTPUTS + "__EXECUTION_ORDER", + TABLE_NODE_OUTPUTS + "__" + RUN_ID, "==", runId, PixelDataType.CONST_STRING)); + qs.addOrderBy(TABLE_NODE_OUTPUTS + "__" + EXECUTION_ORDER, QueryColumnOrderBySelector.ORDER_BY_DIRECTION.ASC.toString()); List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); @@ -797,24 +846,21 @@ public static List> getNodeOutputsForRun(String runId) { private static void createAutomationRunsTable(Connection conn, AbstractSqlQueryUtil queryUtil, String database, String schema, boolean allowIfExists, String dateTimeType, String clobType) throws SQLException { - String tableName = AutomationConstants.TABLE_AUTOMATION_RUNS; + String tableName = TABLE_AUTOMATION_RUNS; if (!allowIfExists && queryUtil.tableExists(conn, tableName, database, schema)) { return; } - String[] colNames = { "RUN_ID", "PROJECT_ID", "AUTOMATION_ID", "STATUS", "TRIGGER_TYPE", - "STARTED_AT", "COMPLETED_AT", "FAILED_NODE_ID", - "ERROR_MESSAGE", "LAST_HEARTBEAT", "TOTAL_NODES", "COMPLETED_NODES", "CREATED_BY", - "CANCEL_REQUESTED" }; - String[] types = { "VARCHAR(255)", "VARCHAR(255)", "VARCHAR(255)", "VARCHAR(50)", "VARCHAR(50)", - dateTimeType, dateTimeType, "VARCHAR(255)", - clobType, dateTimeType, "INTEGER", "INTEGER", "VARCHAR(255)", - queryUtil.getBooleanDataTypeName() }; - String[] constraints = { "NOT NULL", "NOT NULL", null, "NOT NULL", "NOT NULL", - "NOT NULL", null, null, - null, null, null, null, null, - null }; + String[] colNames = { RUN_ID, PROJECT_ID, AUTOMATION_ID, STATUS, TRIGGER_TYPE, + STARTED_AT, COMPLETED_AT, FAILED_NODE_ID, ERROR_MESSAGE, LAST_HEARTBEAT, + TOTAL_NODES, COMPLETED_NODES, CREATED_BY, CANCEL_REQUESTED }; + String[] types = { VARCHAR_255, VARCHAR_255, VARCHAR_255, VARCHAR_50, VARCHAR_50, + dateTimeType, dateTimeType, VARCHAR_255, clobType, dateTimeType, + INTEGER, INTEGER, VARCHAR_255, queryUtil.getBooleanDataTypeName() }; + String[] constraints = { NOT_NULL, NOT_NULL, null, NOT_NULL, NOT_NULL, + NOT_NULL, null, null, null, null, + null, null, null, null }; String sql; if (allowIfExists) { @@ -828,33 +874,37 @@ private static void createAutomationRunsTable(Connection conn, AbstractSqlQueryU } // Migrate installs that predate cluster-safe cancel - addColumnIfNotExists(conn, queryUtil, tableName, "CANCEL_REQUESTED", queryUtil.getBooleanDataTypeName()); + addColumnIfNotExists(conn, queryUtil, tableName, CANCEL_REQUESTED, queryUtil.getBooleanDataTypeName()); // Primary key - addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, "PK_AUTOMATION_RUNS", new String[]{"RUN_ID"}); + addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, PK_AUTOMATION_RUNS, + new String[]{ RUN_ID }); // Indexes - createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_AR_PROJECT", tableName, new String[]{"PROJECT_ID"}); - createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_AR_STATUS", tableName, new String[]{"PROJECT_ID", "STATUS"}); - createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_AR_STARTED", tableName, new String[]{"PROJECT_ID", "STARTED_AT"}); + createIndexIfNotExists(conn, queryUtil, allowIfExists, IDX_AR_PROJECT, tableName, + new String[]{ PROJECT_ID }); + createIndexIfNotExists(conn, queryUtil, allowIfExists, IDX_AR_STATUS, tableName, + new String[]{ PROJECT_ID, STATUS }); + createIndexIfNotExists(conn, queryUtil, allowIfExists, IDX_AR_STARTED, tableName, + new String[]{ PROJECT_ID, STARTED_AT }); } private static void createAutomationNodeOutputsTable(Connection conn, AbstractSqlQueryUtil queryUtil, String database, String schema, boolean allowIfExists, String dateTimeType, String clobType) throws SQLException { - String tableName = AutomationConstants.TABLE_AUTOMATION_NODE_OUTPUTS; + String tableName = TABLE_AUTOMATION_NODE_OUTPUTS; if (!allowIfExists && queryUtil.tableExists(conn, tableName, database, schema)) { return; } - String[] colNames = { "RUN_ID", "NODE_ID", "NODE_LABEL", "EXECUTION_ORDER", "STATUS", - "STARTED_AT", "COMPLETED_AT", "DURATION_MS", "OUTPUT_VAR", - "OUTPUT_VALUE", "OUTPUT_PREVIEW", "ERROR_MESSAGE" }; - String[] types = { "VARCHAR(255)", "VARCHAR(255)", "VARCHAR(500)", "INTEGER", "VARCHAR(50)", - dateTimeType, dateTimeType, "BIGINT", "VARCHAR(255)", - clobType, "VARCHAR(2000)", clobType }; - String[] constraints = { "NOT NULL", "NOT NULL", null, "NOT NULL", "NOT NULL", + String[] colNames = { RUN_ID, NODE_ID, NODE_LABEL, EXECUTION_ORDER, STATUS, + STARTED_AT, COMPLETED_AT, DURATION_MS, OUTPUT_VAR, + OUTPUT_VALUE, OUTPUT_PREVIEW, ERROR_MESSAGE }; + String[] types = { VARCHAR_255, VARCHAR_255, VARCHAR_500, INTEGER, VARCHAR_50, + dateTimeType, dateTimeType, BIGINT, VARCHAR_255, + clobType, VARCHAR_2000, clobType }; + String[] constraints = { NOT_NULL, NOT_NULL, null, NOT_NULL, NOT_NULL, null, null, null, null, null, null, null }; @@ -870,10 +920,12 @@ private static void createAutomationNodeOutputsTable(Connection conn, AbstractSq } // Composite primary key - addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, "PK_AUTO_NODE_OUT", new String[]{"RUN_ID", "NODE_ID"}); + addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, PK_AUTO_NODE_OUT, + new String[]{ RUN_ID, NODE_ID }); // Indexes - createIndexIfNotExists(conn, queryUtil, allowIfExists, "IDX_ANO_RUN", tableName, new String[]{"RUN_ID"}); + createIndexIfNotExists(conn, queryUtil, allowIfExists, IDX_ANO_RUN, tableName, + new String[]{ RUN_ID }); } /** @@ -884,15 +936,15 @@ private static void createAutomationNodeOutputsTable(Connection conn, AbstractSq private static void createAutomationActiveRunTable(Connection conn, AbstractSqlQueryUtil queryUtil, String database, String schema, boolean allowIfExists, String dateTimeType) throws SQLException { - String tableName = AutomationConstants.TABLE_AUTOMATION_ACTIVE_RUN; + String tableName = TABLE_AUTOMATION_ACTIVE_RUN; if (!allowIfExists && queryUtil.tableExists(conn, tableName, database, schema)) { return; } - String[] colNames = { "PROJECT_ID", "RUN_ID", "CLAIMED_AT" }; - String[] types = { "VARCHAR(255)", "VARCHAR(255)", dateTimeType }; - String[] constraints = { "NOT NULL", "NOT NULL", "NOT NULL" }; + String[] colNames = { PROJECT_ID, RUN_ID, CLAIMED_AT }; + String[] types = { VARCHAR_255, VARCHAR_255, dateTimeType }; + String[] constraints = { NOT_NULL, NOT_NULL, NOT_NULL }; String sql; if (allowIfExists) { @@ -907,7 +959,8 @@ private static void createAutomationActiveRunTable(Connection conn, AbstractSqlQ // Primary key on PROJECT_ID alone (not RUN_ID) is what makes claimActiveRun atomic: // a second INSERT for the same project - from any pod - violates this constraint. - addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, "PK_AUTO_ACTIVE_RUN", new String[]{"PROJECT_ID"}); + addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, PK_AUTO_ACTIVE_RUN, + new String[]{ PROJECT_ID }); } // -- Helpers ------------------------------------------------------------------- diff --git a/src/prerna/reactor/automation/AutomationExecutionUtils.java b/src/prerna/reactor/automation/AutomationExecutionUtils.java index 5506864137d..dbd0647c753 100644 --- a/src/prerna/reactor/automation/AutomationExecutionUtils.java +++ b/src/prerna/reactor/automation/AutomationExecutionUtils.java @@ -39,6 +39,7 @@ import java.util.List; import java.util.Map; +import org.apache.commons.text.StringSubstitutor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -59,6 +60,9 @@ public final class AutomationExecutionUtils { private static final Logger classLogger = LogManager.getLogger(AutomationExecutionUtils.class); + /** Prefix for config-map lookups within a template, e.g. {@code ${config.API_KEY}}. */ + private static final String CONFIG_VAR_PREFIX = "config."; + /** * Shared Gson instance for the whole automation engine — public so the * {@code nodes} sub-package has one shared instance to reuse instead of each @@ -70,20 +74,28 @@ private AutomationExecutionUtils() {} /** * Resolves {@code ${varName}} and {@code ${config.KEY}} placeholders in a template string - * via plain {@link String#replace} — no validation or escaping is applied. + * using {@link StringSubstitutor} (the platform's standard {@code ${...}} templating helper, + * also used by {@code AbstractPythonModelEngine.fillVars}). Recursive substitution is + * disabled so a resolved value is never itself re-scanned for placeholders, and unresolved + * placeholders are left untouched rather than throwing. */ public static String resolve(String template, Map scope, Map configMap) { if (template == null) return ""; - String result = template; + + Map vars = new HashMap<>(); for (Map.Entry e : configMap.entrySet()) { - result = result.replace("${config." + e.getKey() + "}", e.getValue()); + vars.put(CONFIG_VAR_PREFIX + e.getKey(), e.getValue()); } for (Map.Entry e : scope.entrySet()) { if (e.getValue() != null) { - result = result.replace("${" + e.getKey() + "}", e.getValue()); + vars.put(e.getKey(), e.getValue()); } } - return result; + + StringSubstitutor sub = new StringSubstitutor(vars); + sub.setEnableUndefinedVariableException(false); + sub.setDisableSubstitutionInValues(true); + return sub.replace(template); } /** @@ -91,7 +103,7 @@ public static String resolve(String template, Map scope, Map node) { - Object timeout = node.get("timeoutSeconds"); + Object timeout = node.get(AutomationConstants.CONFIG_TIMEOUT_SECONDS); if (timeout instanceof Number) { return ((Number) timeout).intValue(); } @@ -114,8 +126,8 @@ public static Map loadConfig(String projectId) { new TypeToken>>() {}.getType()); if (entries != null) { for (Map entry : entries) { - String key = (String) entry.get("key"); - String value = (String) entry.get("value"); + String key = (String) entry.get(AutomationConstants.CONFIG_ENTRY_KEY); + String value = (String) entry.get(AutomationConstants.CONFIG_ENTRY_VALUE); if (key != null && value != null) map.put(key, value); } } @@ -136,12 +148,12 @@ public static String applyOutputTransform(Object rawResult, Map String rawStr = serializeRaw(rawResult); if (transformConfig == null) return rawStr; - String mode = (String) transformConfig.getOrDefault("mode", "raw"); + String mode = (String) transformConfig.getOrDefault(AutomationConstants.TRANSFORM_MODE, AutomationConstants.TRANSFORM_MODE_RAW); switch (mode) { - case "rows-as-objects": return transformRowsAsObjects(rawStr); - case "first-row": return transformFirstRow(rawStr); - case "column": return transformColumn(rawStr, (String) transformConfig.get("column")); - case "jsonpath": return transformJsonPath(rawStr, (String) transformConfig.get("path")); + case AutomationConstants.TRANSFORM_MODE_ROWS_AS_OBJECTS: return transformRowsAsObjects(rawStr); + case AutomationConstants.TRANSFORM_MODE_FIRST_ROW: return transformFirstRow(rawStr); + case AutomationConstants.TRANSFORM_MODE_COLUMN: return transformColumn(rawStr, (String) transformConfig.get(AutomationConstants.TRANSFORM_COLUMN)); + case AutomationConstants.TRANSFORM_MODE_JSONPATH: return transformJsonPath(rawStr, (String) transformConfig.get(AutomationConstants.TRANSFORM_PATH)); default: return rawStr; } } @@ -159,8 +171,8 @@ public static String serializeRaw(Object rawResult) { private static String transformRowsAsObjects(String rawStr) { Map data = extractDataset(parseJsonAny(rawStr)); if (data == null) return rawStr; - List headers = (List) data.get("headers"); - List> rows = (List>) data.get("values"); + List headers = (List) data.get(AutomationConstants.DATASET_HEADERS); + List> rows = (List>) data.get(AutomationConstants.DATASET_VALUES); if (headers == null || rows == null) return rawStr; List> result = new ArrayList<>(); for (List row : rows) { @@ -177,8 +189,8 @@ private static String transformRowsAsObjects(String rawStr) { private static String transformFirstRow(String rawStr) { Map data = extractDataset(parseJsonAny(rawStr)); if (data == null) return rawStr; - List headers = (List) data.get("headers"); - List> rows = (List>) data.get("values"); + List headers = (List) data.get(AutomationConstants.DATASET_HEADERS); + List> rows = (List>) data.get(AutomationConstants.DATASET_VALUES); if (headers == null || rows == null || rows.isEmpty()) return rawStr; Map rowMap = new HashMap<>(); List first = rows.get(0); @@ -193,8 +205,8 @@ private static String transformColumn(String rawStr, String colName) { if (colName == null || colName.isEmpty()) return rawStr; Map data = extractDataset(parseJsonAny(rawStr)); if (data == null) return rawStr; - List headers = (List) data.get("headers"); - List> rows = (List>) data.get("values"); + List headers = (List) data.get(AutomationConstants.DATASET_HEADERS); + List> rows = (List>) data.get(AutomationConstants.DATASET_VALUES); if (headers == null || rows == null) return rawStr; int colIdx = headers.indexOf(colName); if (colIdx < 0) return rawStr; @@ -246,8 +258,8 @@ private static Map extractDataset(Object parsed) { } } Map result = new HashMap<>(); - result.put("headers", headers); - result.put("values", values); + result.put(AutomationConstants.DATASET_HEADERS, headers); + result.put(AutomationConstants.DATASET_VALUES, values); return result; } @@ -255,11 +267,11 @@ private static Map extractDataset(Object parsed) { Map map = (Map) parsed; // Format 2: {data: {headers, values}} - if (map.containsKey("data") && map.get("data") instanceof Map) { - return (Map) map.get("data"); + if (map.containsKey(AutomationConstants.DATASET_DATA) && map.get(AutomationConstants.DATASET_DATA) instanceof Map) { + return (Map) map.get(AutomationConstants.DATASET_DATA); } // Format 3: {headers, values} - if (map.containsKey("headers") && map.containsKey("values")) return map; + if (map.containsKey(AutomationConstants.DATASET_HEADERS) && map.containsKey(AutomationConstants.DATASET_VALUES)) return map; return null; } @@ -297,9 +309,9 @@ public static Map buildInitialScope(String runId, User user) { Map scope = new HashMap<>(); ZoneId zone = (user != null && user.getZoneId() != null) ? user.getZoneId() : ZoneId.of("UTC"); ZonedDateTime now = ZonedDateTime.now(zone); - scope.put("date", now.format(DateTimeFormatter.ISO_LOCAL_DATE)); - scope.put("triggered_at", now.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - if (runId != null && !runId.isBlank()) scope.put("run_id", runId); + scope.put(AutomationConstants.SCOPE_DATE, now.format(DateTimeFormatter.ISO_LOCAL_DATE)); + scope.put(AutomationConstants.SCOPE_TRIGGERED_AT, now.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + if (runId != null && !runId.isBlank()) scope.put(AutomationConstants.SCOPE_RUN_ID, runId); return scope; } diff --git a/src/prerna/reactor/automation/AutomationRunEngine.java b/src/prerna/reactor/automation/AutomationRunEngine.java index 32f47d85ba2..85d649c875c 100644 --- a/src/prerna/reactor/automation/AutomationRunEngine.java +++ b/src/prerna/reactor/automation/AutomationRunEngine.java @@ -99,10 +99,10 @@ public static void run(String runId, String projectId, try { for (Map node : ordered) { - String nodeId = (String) node.get("id"); - String nodeLabel = (String) node.get("label"); - String outputVar = (String) node.get("outputVar"); - String nodeType = (String) node.get("type"); + String nodeId = (String) node.get(AutomationConstants.NODE_FIELD_ID); + String nodeLabel = (String) node.get(AutomationConstants.NODE_FIELD_LABEL); + String outputVar = (String) node.get(AutomationConstants.NODE_FIELD_OUTPUT_VAR); + String nodeType = (String) node.get(AutomationConstants.NODE_FIELD_TYPE); if (cancelled.get() || AutomationDatabaseUtility.isCancelRequested(runId)) { classLogger.info("Automation run {} cancelled before node {} ({})", runId, nodeId, nodeLabel); @@ -126,7 +126,7 @@ public static void run(String runId, String projectId, if (AutomationConstants.NODE_STATUS_SUCCESS.equals(status)) { if (outputVar != null && !outputVar.isEmpty() && !AutomationConstants.NODE_TRIGGER.equals(nodeType)) { - String outputValue = (String) nodeResult.get("outputValue"); + String outputValue = (String) nodeResult.get(AutomationConstants.RESULT_OUTPUT_VALUE); scope.put(outputVar, outputValue != null ? outputValue : ""); } completedCount++; @@ -156,14 +156,14 @@ private static Map executeSingleNode(String runId, String projec Map node, Map scope, Map configMap, AtomicBoolean cancelFlag, Insight insight) { - String nodeId = (String) node.get("id"); - String nodeLabel = (String) node.get("label"); - String outputVar = (String) node.get("outputVar"); - String type = (String) node.get("type"); + String nodeId = (String) node.get(AutomationConstants.NODE_FIELD_ID); + String nodeLabel = (String) node.get(AutomationConstants.NODE_FIELD_LABEL); + String outputVar = (String) node.get(AutomationConstants.NODE_FIELD_OUTPUT_VAR); + String type = (String) node.get(AutomationConstants.NODE_FIELD_TYPE); if (AutomationConstants.NODE_TRIGGER.equals(type)) { return buildNodeResult(nodeId, nodeLabel, AutomationConstants.NODE_STATUS_SUCCESS, 0, - scope.get("triggered_at"), null); + scope.get(AutomationConstants.SCOPE_TRIGGERED_AT), null); } classLogger.debug("Executing node {} ({}) type={} in run {}", nodeId, nodeLabel, type, runId); @@ -182,7 +182,7 @@ private static Map executeSingleNode(String runId, String projec Object rawOutput = executor.execute(ctx); @SuppressWarnings("unchecked") - Map transformConfig = (Map) node.get("outputTransform"); + Map transformConfig = (Map) node.get(AutomationConstants.NODE_FIELD_OUTPUT_TRANSFORM); String transformed = AutomationExecutionUtils.applyOutputTransform(rawOutput, transformConfig); long durationMs = System.currentTimeMillis() - startMs; String preview = AutomationExecutionUtils.generatePreview(transformed); @@ -192,7 +192,7 @@ private static Map executeSingleNode(String runId, String projec classLogger.debug("Node {} ({}) succeeded in {}ms in run {}", nodeId, nodeLabel, durationMs, runId); Map result = buildNodeResult(nodeId, nodeLabel, AutomationConstants.NODE_STATUS_SUCCESS, durationMs, preview, null); - result.put("outputValue", transformed); + result.put(AutomationConstants.RESULT_OUTPUT_VALUE, transformed); return result; } catch (AutomationCancelledException ace) { diff --git a/src/prerna/reactor/automation/CancelAutomationRunReactor.java b/src/prerna/reactor/automation/CancelAutomationRunReactor.java index df1bb097f65..0d22c1f2988 100644 --- a/src/prerna/reactor/automation/CancelAutomationRunReactor.java +++ b/src/prerna/reactor/automation/CancelAutomationRunReactor.java @@ -35,8 +35,10 @@ import prerna.auth.utils.SecurityProjectUtils; import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.mcp.MCPUtility; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; import prerna.sablecc2.om.nounmeta.NounMetadata; /** @@ -58,8 +60,12 @@ public class CancelAutomationRunReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(CancelAutomationRunReactor.class); + // Not standardized in ReactorKeysEnum — matches the local-key convention used by + // prerna.reactor.agent (e.g. StopAgentRunReactor.RUN_ID_KEY). + private static final String RUN_ID_KEY = "runId"; + public CancelAutomationRunReactor() { - this.keysToGet = new String[] { "project", "runId" }; + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), RUN_ID_KEY }; this.keyRequired = new int[] { 1, 1 }; } @@ -82,9 +88,11 @@ public NounMetadata execute() { throw new IllegalArgumentException("Project does not exist or user does not have edit access"); } - // Validate the run exists and is running + // Validate the run exists, belongs to this project, and is running. Scoping by + // PROJECT_ID prevents a user with edit access to their own project from cancelling + // a run that belongs to a project they were never granted access to. Map runDetail = AutomationDatabaseUtility.getRunDetail(runId); - if (runDetail == null) { + if (runDetail == null || !projectId.equals(runDetail.get(AutomationConstants.PROJECT_ID))) { throw new IllegalArgumentException("Run not found: " + runId); } @@ -110,8 +118,21 @@ public NounMetadata execute() { Map result = new HashMap<>(); result.put(AutomationConstants.RUN_ID, runId); - result.put("cancelRequested", true); - result.put("signalledLocally", signalledLocally); + result.put(AutomationConstants.RESULT_CANCEL_REQUESTED, true); + result.put(AutomationConstants.RESULT_SIGNALLED_LOCALLY, signalledLocally); return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); } + + @Override + public String getReactorDescription() { + return "Requests cancellation of a running automation run for the given project."; + } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + // Cancelling a run is a mutating, side-effecting action — requires explicit confirmation. + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + return meta; + } } diff --git a/src/prerna/reactor/automation/GetActiveAutomationRunReactor.java b/src/prerna/reactor/automation/GetActiveAutomationRunReactor.java index cd85e12d0f7..d977d5bb0b4 100644 --- a/src/prerna/reactor/automation/GetActiveAutomationRunReactor.java +++ b/src/prerna/reactor/automation/GetActiveAutomationRunReactor.java @@ -37,6 +37,7 @@ import prerna.reactor.AbstractReactor; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; import prerna.sablecc2.om.nounmeta.NounMetadata; /** @@ -57,7 +58,7 @@ public class GetActiveAutomationRunReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(GetActiveAutomationRunReactor.class); public GetActiveAutomationRunReactor() { - this.keysToGet = new String[] { "project" }; + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey() }; this.keyRequired = new int[] { 1 }; } @@ -95,7 +96,7 @@ public String getReactorDescription() { @Override protected String getDescriptionForKey(String key) { - if ("project".equals(key)) return "The project (app) ID or alias to check for an active run."; + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) return "The project (app) ID or alias to check for an active run."; return super.getDescriptionForKey(key); } } diff --git a/src/prerna/reactor/automation/GetAutomationConfigReactor.java b/src/prerna/reactor/automation/GetAutomationConfigReactor.java index 4ce339b0fe4..ed2f99cae1d 100644 --- a/src/prerna/reactor/automation/GetAutomationConfigReactor.java +++ b/src/prerna/reactor/automation/GetAutomationConfigReactor.java @@ -44,6 +44,7 @@ import prerna.reactor.AbstractReactor; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; import prerna.sablecc2.om.nounmeta.NounMetadata; import prerna.util.AssetUtility; @@ -65,7 +66,7 @@ public class GetAutomationConfigReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(GetAutomationConfigReactor.class); public GetAutomationConfigReactor() { - this.keysToGet = new String[] { "project" }; + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey() }; } @Override @@ -94,9 +95,9 @@ public NounMetadata execute() { List> entries = AutomationExecutionUtils.GSON.fromJson(json, new TypeToken>>() {}.getType()); if (entries != null) { for (Map entry : entries) { - Object sensitive = entry.get("sensitive"); + Object sensitive = entry.get(AutomationConstants.CONFIG_ENTRY_SENSITIVE); if (Boolean.TRUE.equals(sensitive)) { - entry.put("value", AutomationConstants.SENSITIVE_MASK); + entry.put(AutomationConstants.CONFIG_ENTRY_VALUE, AutomationConstants.SENSITIVE_MASK); } } } diff --git a/src/prerna/reactor/automation/GetAutomationReactor.java b/src/prerna/reactor/automation/GetAutomationReactor.java index 7b6f52e95a3..bf2170f8eaf 100644 --- a/src/prerna/reactor/automation/GetAutomationReactor.java +++ b/src/prerna/reactor/automation/GetAutomationReactor.java @@ -45,6 +45,7 @@ import prerna.reactor.AbstractReactor; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; import prerna.sablecc2.om.nounmeta.NounMetadata; import prerna.util.AssetUtility; import prerna.util.Utility; @@ -70,7 +71,7 @@ public class GetAutomationReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(GetAutomationReactor.class); public GetAutomationReactor() { - this.keysToGet = new String[] { "project" }; + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey() }; } @Override @@ -98,11 +99,11 @@ public NounMetadata execute() { if (!automationFile.exists() || !automationFile.isFile()) { // return empty graph document for brand-new automations Map empty = new HashMap<>(); - empty.put("version", 1); + empty.put(AutomationConstants.DOC_VERSION, AutomationConstants.DOC_CURRENT_VERSION); Map graph = new HashMap<>(); - graph.put("nodes", new ArrayList<>()); - graph.put("edges", new ArrayList<>()); - empty.put("graph", graph); + graph.put(AutomationConstants.DOC_NODES, new ArrayList<>()); + graph.put(AutomationConstants.DOC_EDGES, new ArrayList<>()); + empty.put(AutomationConstants.DOC_GRAPH, graph); return new NounMetadata(empty, PixelDataType.MAP, PixelOperationType.OPERATION); } diff --git a/src/prerna/reactor/automation/GetAutomationRunReactor.java b/src/prerna/reactor/automation/GetAutomationRunReactor.java index 4d66cbeeb68..85bf87e9fc6 100644 --- a/src/prerna/reactor/automation/GetAutomationRunReactor.java +++ b/src/prerna/reactor/automation/GetAutomationRunReactor.java @@ -36,6 +36,7 @@ import prerna.reactor.AbstractReactor; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; import prerna.sablecc2.om.nounmeta.NounMetadata; /** @@ -47,8 +48,12 @@ */ public class GetAutomationRunReactor extends AbstractReactor { + // Not standardized in ReactorKeysEnum — matches the local-key convention used by + // prerna.reactor.agent (e.g. GetAgentRunReactor.RUN_ID_KEY). + private static final String RUN_ID_KEY = "runId"; + public GetAutomationRunReactor() { - this.keysToGet = new String[]{ "project", "runId" }; + this.keysToGet = new String[]{ ReactorKeysEnum.PROJECT.getKey(), RUN_ID_KEY }; this.keyRequired = new int[]{ 1, 1 }; } @@ -71,10 +76,12 @@ public NounMetadata execute() { } Map runDetail = AutomationDatabaseUtility.getRunDetail(runId); - if (runDetail == null) { + // Scope by PROJECT_ID so a user with view access to one project cannot read another + // project's run detail/node outputs by guessing or reusing a runId. + if (runDetail == null || !projectId.equals(runDetail.get(AutomationConstants.PROJECT_ID))) { Map notFound = new HashMap<>(); notFound.put(AutomationConstants.RUN_ID, runId); - notFound.put("nodeResults", new ArrayList<>()); + notFound.put(AutomationConstants.RESULT_NODE_RESULTS, new ArrayList<>()); return new NounMetadata(notFound, PixelDataType.MAP, PixelOperationType.OPERATION); } @@ -92,7 +99,12 @@ public NounMetadata execute() { nodeResults.add(nodeResult); } - runDetail.put("nodeResults", nodeResults); + runDetail.put(AutomationConstants.RESULT_NODE_RESULTS, nodeResults); return new NounMetadata(runDetail, PixelDataType.MAP, PixelOperationType.OPERATION); } + + @Override + public String getReactorDescription() { + return "Returns detail for a single automation run, including per-node results."; + } } diff --git a/src/prerna/reactor/automation/ListAutomationRunsReactor.java b/src/prerna/reactor/automation/ListAutomationRunsReactor.java index b53f8c0efb8..c4ad3ef4676 100644 --- a/src/prerna/reactor/automation/ListAutomationRunsReactor.java +++ b/src/prerna/reactor/automation/ListAutomationRunsReactor.java @@ -35,6 +35,7 @@ import prerna.reactor.AbstractReactor; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; import prerna.sablecc2.om.nounmeta.NounMetadata; /** @@ -47,7 +48,7 @@ public class ListAutomationRunsReactor extends AbstractReactor { public ListAutomationRunsReactor() { - this.keysToGet = new String[]{ "project", "limit" }; + this.keysToGet = new String[]{ ReactorKeysEnum.PROJECT.getKey(), ReactorKeysEnum.LIMIT.getKey() }; this.keyRequired = new int[]{ 1, 0 }; } @@ -72,11 +73,16 @@ public NounMetadata execute() { } private int parseLimit(String limitStr) { - if (limitStr == null || limitStr.isEmpty()) return 25; + if (limitStr == null || limitStr.isEmpty()) return AutomationConstants.DEFAULT_LIST_RUNS_LIMIT; try { return Integer.parseInt(limitStr.trim()); } catch (NumberFormatException e) { - return 25; + return AutomationConstants.DEFAULT_LIST_RUNS_LIMIT; } } + + @Override + public String getReactorDescription() { + return "Lists automation run history for a project, newest first."; + } } diff --git a/src/prerna/reactor/automation/RunAutomationNodeReactor.java b/src/prerna/reactor/automation/RunAutomationNodeReactor.java index d5dab53409c..1951dd8b4ee 100644 --- a/src/prerna/reactor/automation/RunAutomationNodeReactor.java +++ b/src/prerna/reactor/automation/RunAutomationNodeReactor.java @@ -37,11 +37,13 @@ import prerna.auth.utils.SecurityProjectUtils; import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.mcp.MCPUtility; import prerna.reactor.automation.nodes.AutomationNodeContext; import prerna.reactor.automation.nodes.AutomationNodeExecutors; import prerna.reactor.automation.nodes.IAutomationNodeExecutor; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; import prerna.sablecc2.om.nounmeta.NounMetadata; /** @@ -54,8 +56,13 @@ public class RunAutomationNodeReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(RunAutomationNodeReactor.class); + // Not standardized in ReactorKeysEnum — matches the local-key convention used by + // prerna.reactor.agent (e.g. GetAgentRunReactor.RUN_ID_KEY). + private static final String NODE_ID_KEY = "nodeId"; + private static final String RUN_ID_KEY = "runId"; + public RunAutomationNodeReactor() { - this.keysToGet = new String[] { "project", "nodeId", "runId" }; + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), NODE_ID_KEY, RUN_ID_KEY }; this.keyRequired = new int[] { 1, 1, 0 }; } @@ -83,29 +90,29 @@ public NounMetadata execute() { throw new IllegalArgumentException("Node not found in automation: " + nodeId); } - Map scope = buildScope(contextRunId); + Map scope = buildScope(projectId, contextRunId); Map configMap = AutomationExecutionUtils.loadConfig(projectId); long startMs = System.currentTimeMillis(); try { - String type = (String) node.get("type"); + String type = (String) node.get(AutomationConstants.NODE_FIELD_TYPE); Object rawOutput; if (AutomationConstants.NODE_TRIGGER.equals(type)) { - rawOutput = scope.get("triggered_at"); + rawOutput = scope.get(AutomationConstants.SCOPE_TRIGGERED_AT); } else { IAutomationNodeExecutor executor = AutomationNodeExecutors.EXECUTORS.get(type); if (executor == null) { throw new IllegalArgumentException("Unsupported node type: " + type); } AutomationNodeContext ctx = new AutomationNodeContext( - "test", projectId, node, scope, configMap, + AutomationConstants.TEST_RUN_ID, projectId, node, scope, configMap, this.insight, new AtomicBoolean(false)); rawOutput = executor.execute(ctx); } @SuppressWarnings("unchecked") - Map transformConfig = (Map) node.get("outputTransform"); + Map transformConfig = (Map) node.get(AutomationConstants.NODE_FIELD_OUTPUT_TRANSFORM); String transformed = AutomationExecutionUtils.applyOutputTransform(rawOutput, transformConfig); long durationMs = System.currentTimeMillis() - startMs; String preview = AutomationExecutionUtils.generatePreview(transformed); @@ -134,20 +141,28 @@ public NounMetadata execute() { @SuppressWarnings("unchecked") private static Map findNode(String projectId, String nodeId) { Map doc = AutomationExecutionUtils.loadAutomationDoc(projectId); - Map graph = (Map) doc.get("graph"); - List> nodes = (List>) graph.get("nodes"); + Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); + List> nodes = (List>) graph.get(AutomationConstants.DOC_NODES); if (nodes != null) { for (Map node : nodes) { - if (nodeId.equals(node.get("id"))) return node; + if (nodeId.equals(node.get(AutomationConstants.NODE_FIELD_ID))) return node; } } return null; } - private Map buildScope(String contextRunId) { + private Map buildScope(String projectId, String contextRunId) { Map scope = AutomationExecutionUtils.buildInitialScope(null, this.insight.getUser()); if (contextRunId != null && !contextRunId.isEmpty()) { + // Scope the context run to this project — otherwise a user could pull node outputs + // (potentially containing other apps' secrets/data) from a run belonging to a + // project they don't have access to by passing an arbitrary runId. + Map contextRunDetail = AutomationDatabaseUtility.getRunDetail(contextRunId); + if (contextRunDetail == null || !projectId.equals(contextRunDetail.get(AutomationConstants.PROJECT_ID))) { + throw new IllegalArgumentException("Run not found: " + contextRunId); + } + List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(contextRunId); for (Map output : nodeOutputs) { String status = (String) output.get(AutomationConstants.STATUS); @@ -167,4 +182,13 @@ private Map buildScope(String contextRunId) { public String getReactorDescription() { return "Executes a single automation node in isolation for testing — result is not persisted."; } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + // Node executors can perform real side effects (DB writes, storage uploads/deletes, + // arbitrary pixel execution) — requires explicit human confirmation. + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + return meta; + } } diff --git a/src/prerna/reactor/automation/SaveAutomationConfigReactor.java b/src/prerna/reactor/automation/SaveAutomationConfigReactor.java index d66997fde41..e4f2e0b9a64 100644 --- a/src/prerna/reactor/automation/SaveAutomationConfigReactor.java +++ b/src/prerna/reactor/automation/SaveAutomationConfigReactor.java @@ -39,24 +39,23 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; import com.google.gson.reflect.TypeToken; import prerna.auth.utils.SecurityProjectUtils; import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.mcp.MCPUtility; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; import prerna.sablecc2.om.nounmeta.NounMetadata; import prerna.util.AssetUtility; public class SaveAutomationConfigReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(SaveAutomationConfigReactor.class); - private static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); public SaveAutomationConfigReactor() { - this.keysToGet = new String[]{ "project", "config" }; + this.keysToGet = new String[]{ ReactorKeysEnum.PROJECT.getKey(), ReactorKeysEnum.CONFIG.getKey() }; } @Override @@ -76,9 +75,9 @@ public NounMetadata execute() { String config; try { - config = URLDecoder.decode(configEncoded != null ? configEncoded : "[]", StandardCharsets.UTF_8); + config = URLDecoder.decode(configEncoded != null ? configEncoded : AutomationConstants.EMPTY_JSON_ARRAY, StandardCharsets.UTF_8); } catch (Exception e) { - config = configEncoded != null ? configEncoded : "[]"; + config = configEncoded != null ? configEncoded : AutomationConstants.EMPTY_JSON_ARRAY; } String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); @@ -112,10 +111,10 @@ private String restoreMaskedSensitiveValues(String incomingJson, File existingFi return incomingJson; } try { - List> incoming = GSON.fromJson(incomingJson, + List> incoming = AutomationExecutionUtils.GSON.fromJson(incomingJson, new TypeToken>>() {}.getType()); String existingJson = Files.readString(existingFile.toPath(), StandardCharsets.UTF_8); - List> existing = GSON.fromJson(existingJson, + List> existing = AutomationExecutionUtils.GSON.fromJson(existingJson, new TypeToken>>() {}.getType()); if (incoming == null || existing == null || existing.isEmpty()) { return incomingJson; @@ -123,21 +122,21 @@ private String restoreMaskedSensitiveValues(String incomingJson, File existingFi Map existingValueByKey = new HashMap<>(); for (Map e : existing) { - existingValueByKey.put(String.valueOf(e.get("key")), e.get("value")); + existingValueByKey.put(String.valueOf(e.get(AutomationConstants.CONFIG_ENTRY_KEY)), e.get(AutomationConstants.CONFIG_ENTRY_VALUE)); } boolean restoredAny = false; for (Map entry : incoming) { - if (Boolean.TRUE.equals(entry.get("sensitive")) - && AutomationConstants.SENSITIVE_MASK.equals(entry.get("value"))) { - Object real = existingValueByKey.get(String.valueOf(entry.get("key"))); + if (Boolean.TRUE.equals(entry.get(AutomationConstants.CONFIG_ENTRY_SENSITIVE)) + && AutomationConstants.SENSITIVE_MASK.equals(entry.get(AutomationConstants.CONFIG_ENTRY_VALUE))) { + Object real = existingValueByKey.get(String.valueOf(entry.get(AutomationConstants.CONFIG_ENTRY_KEY))); if (real != null) { - entry.put("value", real); + entry.put(AutomationConstants.CONFIG_ENTRY_VALUE, real); restoredAny = true; } } } - return restoredAny ? GSON.toJson(incoming) : incomingJson; + return restoredAny ? AutomationExecutionUtils.GSON.toJson(incoming) : incomingJson; } catch (Exception e) { // Never let a merge problem overwrite good secrets - keep the existing file untouched. classLogger.warn("Could not merge sensitive automation config; keeping existing file. Cause: {}", @@ -149,4 +148,17 @@ private String restoreMaskedSensitiveValues(String incomingJson, File existingFi } } } + + @Override + public String getReactorDescription() { + return "Saves the automation config (key/value env vars and secrets) for a project."; + } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + // Overwrites saved config values/secrets — requires explicit human confirmation. + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + return meta; + } } diff --git a/src/prerna/reactor/automation/SaveAutomationReactor.java b/src/prerna/reactor/automation/SaveAutomationReactor.java index 99da53ea6d6..e965a2a1f82 100644 --- a/src/prerna/reactor/automation/SaveAutomationReactor.java +++ b/src/prerna/reactor/automation/SaveAutomationReactor.java @@ -33,7 +33,9 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -42,8 +44,10 @@ import prerna.cluster.util.ClusterUtil; import prerna.project.api.IProject; import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.mcp.MCPUtility; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; import prerna.sablecc2.om.nounmeta.NounMetadata; import prerna.util.AssetUtility; import prerna.util.Utility; @@ -54,7 +58,7 @@ public class SaveAutomationReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(SaveAutomationReactor.class); public SaveAutomationReactor() { - this.keysToGet = new String[]{ "project", "json" }; + this.keysToGet = new String[]{ ReactorKeysEnum.PROJECT.getKey(), ReactorKeysEnum.JSON.getKey() }; } @Override @@ -115,4 +119,18 @@ public NounMetadata execute() { SecurityProjectUtils.updateProjectLastEditedDate(projectId); return new NounMetadata(true, PixelDataType.BOOLEAN, PixelOperationType.OPERATION); } + + @Override + public String getReactorDescription() { + return "Saves the automation graph (automation.json) for a project."; + } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + // Overwrites the saved automation graph and commits it to source control — + // requires explicit human confirmation. + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + return meta; + } } diff --git a/src/prerna/reactor/automation/TriggerAutomationReactor.java b/src/prerna/reactor/automation/TriggerAutomationReactor.java index f71388d6362..1cf26ba463f 100644 --- a/src/prerna/reactor/automation/TriggerAutomationReactor.java +++ b/src/prerna/reactor/automation/TriggerAutomationReactor.java @@ -42,6 +42,7 @@ import prerna.reactor.agent.mcp.MCPUtility.MCPDisplayOption; import prerna.reactor.agent.mcp.MCPUtility.MCPExecution; import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.ReactorKeysEnum; import prerna.sablecc2.om.PixelOperationType; import prerna.sablecc2.om.nounmeta.NounMetadata; @@ -57,7 +58,7 @@ public class TriggerAutomationReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(TriggerAutomationReactor.class); public TriggerAutomationReactor() { - this.keysToGet = new String[] { "project" }; + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey() }; this.keyRequired = new int[] { 1 }; } @@ -79,9 +80,9 @@ public NounMetadata execute() { try { Map doc = AutomationExecutionUtils.loadAutomationDoc(projectId); @SuppressWarnings("unchecked") - Map graph = (Map) doc.get("graph"); + Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); @SuppressWarnings("unchecked") - List> nodes = (List>) graph.get("nodes"); + List> nodes = (List>) graph.get(AutomationConstants.DOC_NODES); Map configMap = AutomationExecutionUtils.loadConfig(projectId); List> ordered = nodes != null ? nodes : new ArrayList<>(); @@ -118,7 +119,7 @@ public NounMetadata execute() { runDetail.put(AutomationConstants.RUN_ID, runId); runDetail.put(AutomationConstants.PROJECT_ID, projectId); } - runDetail.put("nodeResults", nodeResults); + runDetail.put(AutomationConstants.RESULT_NODE_RESULTS, nodeResults); return new NounMetadata(runDetail, PixelDataType.MAP, PixelOperationType.OPERATION); } catch (Exception e) { @@ -151,7 +152,7 @@ private String getUserId() { if (this.insight.getUser() != null && this.insight.getUser().getPrimaryLoginToken() != null) { return this.insight.getUser().getPrimaryLoginToken().getId(); } - return "system"; + return AutomationConstants.SYSTEM_USER_ID; } @Override @@ -169,7 +170,7 @@ public String getReactorDescription() { @Override protected String getDescriptionForKey(String key) { - if ("project".equals(key)) return "The project (app) ID or alias to run the automation for."; + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) return "The project (app) ID or alias to run the automation for."; return super.getDescriptionForKey(key); } } diff --git a/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java index 0fe96a712f0..285a6e8e952 100644 --- a/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java @@ -32,8 +32,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import prerna.auth.utils.SecurityProjectUtils; import prerna.om.ThreadStore; import prerna.project.api.IProject; +import prerna.reactor.automation.AutomationConstants; import prerna.reactor.automation.AutomationExecutionUtils; import prerna.reactor.automation.PixelExecutionUtils; import prerna.util.Utility; @@ -52,6 +54,10 @@ * before execution. {@link PixelExecutionUtils#runAndCollect} snapshots the caller's thread * context (including these overrides) before submitting to the timeout thread, so the context * propagates correctly even under timeout enforcement. + * + *

The caller must have edit access to {@code appId} — this executor runs an arbitrary pixel + * expression in that project's context, so a view-only check would let any automation editor + * reach into projects they cannot otherwise modify. */ public final class AppEngineNodeExecutor implements IAutomationNodeExecutor { @@ -64,14 +70,19 @@ public Object execute(AutomationNodeContext ctx) throws Exception { Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String pixel = required(config, "pixel", nodeLabel); - String appId = optional(config, "appId"); + String pixel = required(config, AutomationConstants.CONFIG_PIXEL, nodeLabel); + String appId = optional(config, AutomationConstants.CONFIG_APP_ID); String resolvedPixel = AutomationExecutionUtils.resolve(pixel, scope, configMap); String resolvedAppId = appId != null ? AutomationExecutionUtils.resolve(appId, scope, configMap) : null; classLogger.debug("App-engine node \"{}\" executing pixel in appId={}", nodeLabel, resolvedAppId != null ? resolvedAppId : "caller context"); if (resolvedAppId != null && !resolvedAppId.isBlank()) { + resolvedAppId = SecurityProjectUtils.testUserProjectIdForAlias(ctx.insight().getUser(), resolvedAppId); + if (!SecurityProjectUtils.userCanEditProject(ctx.insight().getUser(), resolvedAppId)) { + throw new IllegalArgumentException( + "App node \"" + nodeLabel + "\": project does not exist or user does not have access: " + resolvedAppId); + } IProject project = Utility.getProject(resolvedAppId); if (project == null) { throw new IllegalArgumentException( diff --git a/src/prerna/reactor/automation/nodes/AutomationNodeContext.java b/src/prerna/reactor/automation/nodes/AutomationNodeContext.java index 0ae3fbeeafa..ceca4f5b620 100644 --- a/src/prerna/reactor/automation/nodes/AutomationNodeContext.java +++ b/src/prerna/reactor/automation/nodes/AutomationNodeContext.java @@ -31,6 +31,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import prerna.om.Insight; +import prerna.reactor.automation.AutomationConstants; /** * Immutable param bundle passed to every {@link IAutomationNodeExecutor}. @@ -56,21 +57,21 @@ public record AutomationNodeContext( AtomicBoolean cancelFlag) { public String nodeId() { - return (String) node.get("id"); + return (String) node.get(AutomationConstants.NODE_FIELD_ID); } public String nodeLabel() { - Object label = node.get("label"); - return label != null ? label.toString() : "unnamed"; + Object label = node.get(AutomationConstants.NODE_FIELD_LABEL); + return label != null ? label.toString() : AutomationConstants.UNNAMED_NODE_LABEL; } public String nodeType() { - return (String) node.get("type"); + return (String) node.get(AutomationConstants.NODE_FIELD_TYPE); } @SuppressWarnings("unchecked") public Map config() { - Object config = node.get("config"); + Object config = node.get(AutomationConstants.NODE_FIELD_CONFIG); return config instanceof Map ? (Map) config : Map.of(); } } diff --git a/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java index 8274a18d92e..1d33765e188 100644 --- a/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java @@ -32,6 +32,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import prerna.reactor.automation.AutomationConstants; import prerna.reactor.automation.AutomationExecutionUtils; import prerna.reactor.automation.PixelExecutionUtils; @@ -59,21 +60,22 @@ public Object execute(AutomationNodeContext ctx) throws Exception { Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String engineId = required(config, "engineId", nodeLabel); - String sql = required(config, "expression", nodeLabel); - String operation = optional(config, "operation", "read"); - int limit = optionalInt(config, "limit", 50); + String engineId = required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); + String sql = required(config, AutomationConstants.CONFIG_EXPRESSION, nodeLabel); + String operation = optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_READ); + int limit = optionalInt(config, AutomationConstants.CONFIG_LIMIT, AutomationConstants.DEFAULT_DB_QUERY_LIMIT); String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); String resolvedSql = AutomationExecutionUtils.resolve(sql, scope, configMap); classLogger.debug("Database-engine node \"{}\" executing operation={} via engine {}", nodeLabel, operation, resolvedEngineId); - // Escape double quotes in the SQL for the pixel string literal, then delegate to - // SqlQuery which uses the engine abstraction (HardSelectQueryStruct) and enforces - // the caller's database-level permissions automatically. + // Escape double quotes in the SQL and engine id for the pixel string literal, then + // delegate to SqlQuery which uses the engine abstraction (HardSelectQueryStruct) and + // enforces the caller's database-level permissions automatically. + String escapedEngineId = resolvedEngineId.replace("\"", "\\\""); String escapedSql = resolvedSql.replace("\"", "\\\""); - String pixel = "SqlQuery(database=[\"" + resolvedEngineId + "\"], query=[\"" + escapedSql + "\"], limit=[" + limit + "]);"; + String pixel = "SqlQuery(database=[\"" + escapedEngineId + "\"], query=[\"" + escapedSql + "\"], limit=[" + limit + "]);"; int timeout = AutomationExecutionUtils.getNodeTimeout(ctx.node()); return PixelExecutionUtils.runAndCollect(ctx.insight(), pixel, timeout); diff --git a/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java index c1c0928063a..1cb06b7b276 100644 --- a/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java @@ -34,7 +34,10 @@ import com.google.gson.reflect.TypeToken; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityQueryUtils; import prerna.engine.api.IFunctionEngine; +import prerna.reactor.automation.AutomationConstants; import prerna.reactor.automation.AutomationExecutionUtils; import prerna.util.Utility; @@ -49,12 +52,18 @@ public Object execute(AutomationNodeContext ctx) throws Exception { Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String engineId = required(config, "engineId", nodeLabel); - String params = optional(config, "params", "{}"); + String engineId = required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); + String params = optional(config, AutomationConstants.CONFIG_PARAMS, AutomationConstants.EMPTY_JSON_OBJECT); String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); String resolvedParams = AutomationExecutionUtils.resolve(params, scope, configMap); + resolvedEngineId = SecurityQueryUtils.testUserEngineIdForAlias(ctx.insight().getUser(), resolvedEngineId); + if (!SecurityEngineUtils.userCanViewEngine(ctx.insight().getUser(), resolvedEngineId)) { + throw new IllegalArgumentException( + "Function-engine node \"" + nodeLabel + "\": engine does not exist or user does not have access: " + resolvedEngineId); + } + IFunctionEngine engine = Utility.getFunctionEngine(resolvedEngineId); if (engine == null) { throw new IllegalArgumentException("Function-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); diff --git a/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java index 2c4eeec20a1..5a53a6d8547 100644 --- a/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java @@ -36,9 +36,12 @@ import com.google.gson.reflect.TypeToken; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityQueryUtils; import prerna.engine.api.IModelEngine; import prerna.engine.impl.model.responses.AskModelEngineResponse; import prerna.engine.impl.model.responses.EmbeddingsModelEngineResponse; +import prerna.reactor.automation.AutomationConstants; import prerna.reactor.automation.AutomationExecutionUtils; import prerna.util.Utility; @@ -53,10 +56,16 @@ public Object execute(AutomationNodeContext ctx) throws Exception { Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String engineId = required(config, "engineId", nodeLabel); - String operation = optional(config, "operation", "llm"); + String engineId = required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); + String operation = optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_LLM); String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); + resolvedEngineId = SecurityQueryUtils.testUserEngineIdForAlias(ctx.insight().getUser(), resolvedEngineId); + if (!SecurityEngineUtils.userCanViewEngine(ctx.insight().getUser(), resolvedEngineId)) { + throw new IllegalArgumentException( + "Model-engine node \"" + nodeLabel + "\": engine does not exist or user does not have access: " + resolvedEngineId); + } + IModelEngine engine = Utility.getModel(resolvedEngineId); if (engine == null) { throw new IllegalArgumentException("Model-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); @@ -64,8 +73,8 @@ public Object execute(AutomationNodeContext ctx) throws Exception { classLogger.debug("Model-engine node \"{}\" executing operation={} via engine {}", nodeLabel, operation, resolvedEngineId); switch (operation) { - case "embeddings": { - String values = required(config, "values", nodeLabel); + case AutomationConstants.OP_EMBEDDINGS: { + String values = required(config, AutomationConstants.CONFIG_VALUES, nodeLabel); String resolvedValues = AutomationExecutionUtils.resolve(values, scope, configMap); List valueList = Arrays.asList(resolvedValues.split(",")); EmbeddingsModelEngineResponse response = engine.embeddings(valueList, ctx.insight(), null); @@ -73,9 +82,9 @@ public Object execute(AutomationNodeContext ctx) throws Exception { } default: { // llm (and vision/ner as fallback — both use ask() with the primary command field) - String command = required(config, "command", nodeLabel); + String command = required(config, AutomationConstants.CONFIG_COMMAND, nodeLabel); String resolvedCommand = AutomationExecutionUtils.resolve(command, scope, configMap); - String context = optional(config, "context"); + String context = optional(config, AutomationConstants.CONFIG_CONTEXT); String resolvedContext = (context != null) ? AutomationExecutionUtils.resolve(context, scope, configMap) : null; Map params = parseParams(config, scope, configMap, nodeLabel); @@ -89,7 +98,7 @@ public Object execute(AutomationNodeContext ctx) throws Exception { @SuppressWarnings("unchecked") private static Map parseParams(Map config, Map scope, Map configMap, String nodeLabel) { - String paramValues = optional(config, "paramValues"); + String paramValues = optional(config, AutomationConstants.CONFIG_PARAM_VALUES); if (paramValues == null) return null; String resolved = AutomationExecutionUtils.resolve(paramValues, scope, configMap); try { diff --git a/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java index 578e02d5a4f..d5d753b9425 100644 --- a/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java @@ -34,7 +34,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityQueryUtils; import prerna.engine.api.IStorageEngine; +import prerna.reactor.automation.AutomationConstants; import prerna.reactor.automation.AutomationExecutionUtils; import prerna.util.Utility; @@ -49,10 +52,21 @@ public Object execute(AutomationNodeContext ctx) throws Exception { Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String engineId = required(config, "engineId", nodeLabel); - String operation = optional(config, "operation", "list"); + String engineId = required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); + String operation = optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_LIST); String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); + resolvedEngineId = SecurityQueryUtils.testUserEngineIdForAlias(ctx.insight().getUser(), resolvedEngineId); + boolean mutating = AutomationConstants.OP_UPLOAD.equals(operation) + || AutomationConstants.OP_DELETE.equals(operation); + boolean authorized = mutating + ? SecurityEngineUtils.userCanEditEngine(ctx.insight().getUser(), resolvedEngineId) + : SecurityEngineUtils.userCanViewEngine(ctx.insight().getUser(), resolvedEngineId); + if (!authorized) { + throw new IllegalArgumentException( + "Storage-engine node \"" + nodeLabel + "\": engine does not exist or user does not have access: " + resolvedEngineId); + } + IStorageEngine engine = Utility.getStorage(resolvedEngineId); if (engine == null) { throw new IllegalArgumentException("Storage-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); @@ -60,37 +74,37 @@ public Object execute(AutomationNodeContext ctx) throws Exception { classLogger.debug("Storage-engine node \"{}\" executing operation={} via engine {}", nodeLabel, operation, resolvedEngineId); switch (operation) { - case "download": { - String storagePath = required(config, "storagePath", nodeLabel); - String filePath = required(config, "filePath", nodeLabel); + case AutomationConstants.OP_DOWNLOAD: { + String storagePath = required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); + String filePath = required(config, AutomationConstants.CONFIG_FILE_PATH, nodeLabel); String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); String resolvedFile = AutomationExecutionUtils.resolve(filePath, scope, configMap); engine.copyToLocal(resolvedStorage, resolvedFile); return "Downloaded: " + resolvedStorage; } - case "upload": { - String storagePath = required(config, "storagePath", nodeLabel); - String filePath = required(config, "filePath", nodeLabel); + case AutomationConstants.OP_UPLOAD: { + String storagePath = required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); + String filePath = required(config, AutomationConstants.CONFIG_FILE_PATH, nodeLabel); String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); String resolvedFile = AutomationExecutionUtils.resolve(filePath, scope, configMap); engine.copyToStorage(resolvedFile, resolvedStorage, null); return "Uploaded: " + resolvedFile; } - case "delete": { - String storagePath = required(config, "storagePath", nodeLabel); + case AutomationConstants.OP_DELETE: { + String storagePath = required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); engine.deleteFromStorage(resolvedStorage); return "Deleted: " + resolvedStorage; } - case "read-base64": { - String storagePath = required(config, "storagePath", nodeLabel); + case AutomationConstants.OP_READ_BASE64: { + String storagePath = required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); byte[] bytes = engine.readBlobToMemory(resolvedStorage); return Base64.getEncoder().encodeToString(bytes); } default: { // list - String storagePath = optional(config, "storagePath", "/"); + String storagePath = optional(config, AutomationConstants.CONFIG_STORAGE_PATH, AutomationConstants.DEFAULT_STORAGE_PATH); String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); List files = engine.list(resolvedStorage); return files; diff --git a/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java index 459f6c988e1..ed8c1a9463e 100644 --- a/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java @@ -34,7 +34,10 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityQueryUtils; import prerna.engine.api.IVectorDatabaseEngine; +import prerna.reactor.automation.AutomationConstants; import prerna.reactor.automation.AutomationExecutionUtils; import prerna.util.Utility; @@ -49,10 +52,22 @@ public Object execute(AutomationNodeContext ctx) throws Exception { Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String engineId = required(config, "engineId", nodeLabel); - String operation = optional(config, "operation", "search"); + String engineId = required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); + String operation = optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_SEARCH); String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); + resolvedEngineId = SecurityQueryUtils.testUserEngineIdForAlias(ctx.insight().getUser(), resolvedEngineId); + boolean mutating = AutomationConstants.OP_ADD_FILE.equals(operation) + || AutomationConstants.OP_ADD_CSV.equals(operation) + || AutomationConstants.OP_DELETE.equals(operation); + boolean authorized = mutating + ? SecurityEngineUtils.userCanEditEngine(ctx.insight().getUser(), resolvedEngineId) + : SecurityEngineUtils.userCanViewEngine(ctx.insight().getUser(), resolvedEngineId); + if (!authorized) { + throw new IllegalArgumentException( + "Vector-engine node \"" + nodeLabel + "\": engine does not exist or user does not have access: " + resolvedEngineId); + } + IVectorDatabaseEngine engine = Utility.getVectorDatabase(resolvedEngineId); if (engine == null) { throw new IllegalArgumentException("Vector-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); @@ -60,20 +75,20 @@ public Object execute(AutomationNodeContext ctx) throws Exception { classLogger.debug("Vector-engine node \"{}\" executing operation={} via engine {}", nodeLabel, operation, resolvedEngineId); switch (operation) { - case "add-file": - case "add-csv": { - String filePaths = required(config, "filePath", nodeLabel); + case AutomationConstants.OP_ADD_FILE: + case AutomationConstants.OP_ADD_CSV: { + String filePaths = required(config, AutomationConstants.CONFIG_FILE_PATH, nodeLabel); String resolvedPaths = AutomationExecutionUtils.resolve(filePaths, scope, configMap); List paths = Arrays.asList(resolvedPaths.split(",")); engine.addDocument(paths, null); return "Added " + paths.size() + " file(s)"; } - case "list": { + case AutomationConstants.OP_LIST: { List> docs = engine.listDocuments(null); return docs; } - case "delete": { - String fileNames = required(config, "fileNames", nodeLabel); + case AutomationConstants.OP_DELETE: { + String fileNames = required(config, AutomationConstants.CONFIG_FILE_NAMES, nodeLabel); String resolvedNames = AutomationExecutionUtils.resolve(fileNames, scope, configMap); List names = Arrays.asList(resolvedNames.split(",")); engine.removeDocument(names, null); @@ -81,9 +96,9 @@ public Object execute(AutomationNodeContext ctx) throws Exception { } default: { // search - String command = required(config, "command", nodeLabel); + String command = required(config, AutomationConstants.CONFIG_COMMAND, nodeLabel); String resolvedCommand = AutomationExecutionUtils.resolve(command, scope, configMap); - int limit = optionalInt(config, "limit", 5); + int limit = optionalInt(config, AutomationConstants.CONFIG_LIMIT, AutomationConstants.DEFAULT_VECTOR_SEARCH_LIMIT); List> results = engine.nearestNeighbor(ctx.insight(), resolvedCommand, limit, null); return results; } diff --git a/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java index ba55007219a..ab64225d116 100644 --- a/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java @@ -34,31 +34,33 @@ import org.apache.logging.log4j.Logger; import prerna.reactor.automation.AutomationCancelledException; +import prerna.reactor.automation.AutomationConstants; import prerna.reactor.automation.AutomationDatabaseUtility; import prerna.reactor.automation.AutomationExecutionUtils; /** * Executes a "wait" node: sleeps for the configured number of seconds. * The {@code seconds} value supports {@code ${var}} template substitution. - * Maximum 3600 seconds (1 hour) per invocation. + * Maximum {@value AutomationConstants#WAIT_MAX_SECONDS} seconds (1 hour) per invocation. * - *

Sleeps in 5-second chunks, checking the cancellation flag between each chunk. - * This fixes the bug where a single {@code Thread.sleep(N)} call would block for the full - * duration even after a cancel request was received - the cancel check was only between - * nodes, so a long wait could not be interrupted until it completed naturally. + *

Sleeps in {@value AutomationConstants#WAIT_CANCEL_CHECK_INTERVAL_SECONDS}-second chunks, + * checking the cancellation flag between each chunk. This fixes the bug where a single + * {@code Thread.sleep(N)} call would block for the full duration even after a cancel request + * was received - the cancel check was only between nodes, so a long wait could not be + * interrupted until it completed naturally. */ public final class WaitNodeExecutor implements IAutomationNodeExecutor { private static final Logger classLogger = LogManager.getLogger(WaitNodeExecutor.class); - private static final int CANCEL_CHECK_INTERVAL_SECONDS = 5; @Override public Object execute(AutomationNodeContext ctx) throws Exception { Map config = ctx.config(); String nodeLabel = ctx.nodeLabel(); - String secondsTemplate = config.get("seconds") != null - ? config.get("seconds").toString() : "1"; + String secondsTemplate = config.get(AutomationConstants.CONFIG_SECONDS) != null + ? config.get(AutomationConstants.CONFIG_SECONDS).toString() + : String.valueOf(AutomationConstants.WAIT_DEFAULT_SECONDS); String resolved = AutomationExecutionUtils.resolve(secondsTemplate, ctx.scope(), ctx.configMap()); int seconds; @@ -68,7 +70,7 @@ public Object execute(AutomationNodeContext ctx) throws Exception { throw new IllegalArgumentException("Wait node \"" + nodeLabel + "\" - seconds value is not a valid integer after resolution: \"" + resolved + "\""); } - seconds = Math.min(Math.max(seconds, 0), 3600); + seconds = Math.min(Math.max(seconds, AutomationConstants.WAIT_MIN_SECONDS), AutomationConstants.WAIT_MAX_SECONDS); classLogger.debug("Wait node \"{}\" sleeping {} seconds", nodeLabel, seconds); // Sleep in chunks so cancel requests are honored mid-wait rather than waiting @@ -78,7 +80,7 @@ public Object execute(AutomationNodeContext ctx) throws Exception { if (ctx.cancelFlag().get() || AutomationDatabaseUtility.isCancelRequested(ctx.runId())) { throw new AutomationCancelledException("Wait node \"" + nodeLabel + "\" cancelled"); } - int chunk = Math.min(remaining, CANCEL_CHECK_INTERVAL_SECONDS); + int chunk = Math.min(remaining, AutomationConstants.WAIT_CANCEL_CHECK_INTERVAL_SECONDS); try { TimeUnit.SECONDS.sleep(chunk); } catch (InterruptedException e) { From 306a4155db079a642b570862409ac5cd31ae3c8e Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Thu, 30 Jul 2026 13:00:19 -0400 Subject: [PATCH 12/25] fix: change to asyn and add mcp --- .../automation/AutomationConstants.java | 8 + .../automation/AutomationExecutionUtils.java | 29 +++ .../reactor/automation/AutomationMcpSync.java | 180 ++++++++++++++++++ .../automation/AutomationRunEngine.java | 67 ++++++- .../automation/SaveAutomationReactor.java | 5 + .../automation/TriggerAutomationReactor.java | 28 ++- 6 files changed, 311 insertions(+), 6 deletions(-) create mode 100644 src/prerna/reactor/automation/AutomationMcpSync.java diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index b49cb4a1a71..66981f36afd 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -162,6 +162,12 @@ private AutomationConstants() {} public static final String DOC_NODES = "nodes"; public static final String DOC_EDGES = "edges"; public static final int DOC_CURRENT_VERSION = 1; + /** + * Optional {@code ${var}}/{@code ${config.KEY}} template resolved against the final run + * scope once all nodes complete, producing a workflow-specific human-readable summary + * (e.g. "Indexed 20 files") instead of a raw JSON blob for MCP/agent consumers. + */ + public static final String DOC_RESULT_MESSAGE_TEMPLATE = "resultMessageTemplate"; // -- Node/edge field names -------------------------------------------------------- @@ -209,6 +215,8 @@ private AutomationConstants() {} public static final String RESULT_CANCEL_REQUESTED = "cancelRequested"; public static final String RESULT_SIGNALLED_LOCALLY = "signalledLocally"; public static final String RESULT_OUTPUT_VALUE = "outputValue"; + /** Human-readable, per-workflow summary surfaced to MCP/agent consumers (see {@link #DOC_RESULT_MESSAGE_TEMPLATE}). */ + public static final String RESULT_SUMMARY = "summary"; // -- Pixel execution defaults ---------------------------------------------------- diff --git a/src/prerna/reactor/automation/AutomationExecutionUtils.java b/src/prerna/reactor/automation/AutomationExecutionUtils.java index dbd0647c753..fa9d7930f21 100644 --- a/src/prerna/reactor/automation/AutomationExecutionUtils.java +++ b/src/prerna/reactor/automation/AutomationExecutionUtils.java @@ -315,6 +315,34 @@ public static Map buildInitialScope(String runId, User user) { return scope; } + /** + * Builds the human-readable, per-workflow summary surfaced to MCP/agent consumers as the + * primary tool result (instead of a raw run-detail JSON blob). + * + *

Resolves {@code automation.json}'s optional {@link AutomationConstants#DOC_RESULT_MESSAGE_TEMPLATE} + * (e.g. {@code "Indexed ${file_count} files"}) against the final run scope + config using the + * same {@code ${var}}/{@code ${config.KEY}} substitution as node templates. Falls back to a + * generic completed-nodes message when the template is absent, blank, or resolves to blank. + * + * @param doc the parsed {@code automation.json} document + * @param finalScope the run's scope after all nodes completed (output vars included) + * @param configMap project automation config key-value pairs + * @param completedCount number of nodes that completed successfully + * @param totalCount total number of nodes in the run + */ + public static String buildSummaryMessage(Map doc, Map finalScope, + Map configMap, int completedCount, int totalCount) { + Object rawTemplate = doc != null ? doc.get(AutomationConstants.DOC_RESULT_MESSAGE_TEMPLATE) : null; + String template = strCfg(rawTemplate); + if (template != null) { + String resolved = resolve(template, finalScope, configMap); + if (resolved != null && !resolved.isBlank()) { + return resolved; + } + } + return "Automation completed successfully (" + completedCount + "/" + totalCount + " nodes)."; + } + /** Truncates a string to {@link AutomationConstants#OUTPUT_PREVIEW_MAX_LENGTH} chars. */ public static String generatePreview(String s) { if (s == null) return null; @@ -371,3 +399,4 @@ public static Map loadAutomationDoc(String projectId) { } } + diff --git a/src/prerna/reactor/automation/AutomationMcpSync.java b/src/prerna/reactor/automation/AutomationMcpSync.java new file mode 100644 index 00000000000..e76e6977cd0 --- /dev/null +++ b/src/prerna/reactor/automation/AutomationMcpSync.java @@ -0,0 +1,180 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; + +import prerna.auth.User; +import prerna.cluster.util.ClusterUtil; +import prerna.project.api.IProject; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.reactor.agent.mcp.MCPUtility.MCPDisplayOption; +import prerna.reactor.agent.mcp.MCPUtility.MCPExecution; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.util.AssetUtility; +import prerna.util.Constants; +import prerna.util.git.GitRepoUtils; + +/** + * Keeps each project's own {@code assets/mcp/pixel_mcp.json} in sync with a project-scoped + * {@code TriggerAutomation} MCP tool entry, called by {@link SaveAutomationReactor} on every save + * so the project's automation is always discoverable as an MCP tool without a separate manual + * "make this an MCP tool" step. + * + *

A single-purpose class (not folded into {@link AutomationExecutionUtils}, which is scoped to + * run-execution concerns) so the MCP-catalog-sync responsibility stays isolated and easy to find. + */ +public final class AutomationMcpSync { + + private static final Logger classLogger = LogManager.getLogger(AutomationMcpSync.class); + + /** Stamped into the generated tool as {@link MCPUtility#SMSS_MCP_GENERATOR} so re-saves replace it in place. */ + private static final String AUTOMATION_MCP_GENERATOR_ID = "AutomationMCP"; + + private AutomationMcpSync() { + // utility class + } + + /** + * Writes/updates the project-scoped {@code TriggerAutomation} entry. Uses the same + * merge/generator-stamp helpers as {@code MakePixelMCPReactor} (a distinct generator id), so + * it never disturbs tools the user authored by hand or generated through other flows in the + * same file. + * + *

{@code project} is kept as a required argument on the generated tool (matching the + * existing convention for reactor-scanned MCP tools) rather than hardcoded - the id is still + * fixed to this project by construction of the pixel expression itself, so callers only ever + * need to (re-)confirm which project, never guess a different one. + * + *

Failures here are logged and swallowed - the automation save itself must not fail just + * because the MCP catalog couldn't be refreshed. + * + * @param project the resolved project, or {@code null} if it could not be loaded (e.g. + * deleted concurrently) - a no-op in that case, logged as a warning + * @param projectId the project id (used even when {@code project} is present, for clarity) + * @param user the user performing the save, used as the git commit author + */ + public static void syncTriggerAutomationTool(IProject project, String projectId, User user) { + if (project == null) { + classLogger.warn("Skipping automation MCP tool sync for project {}: project could not be loaded.", + projectId); + return; + } + + try { + JSONArray generated = new JSONArray().put(buildTriggerAutomationTool()); + MCPUtility.stampGenerator(generated, AUTOMATION_MCP_GENERATOR_ID); + + String assetsFolder = AssetUtility.getProjectAssetsFolder(projectId); + String outputFileLoc = assetsFolder + "/mcp/pixel_mcp.json"; + JSONArray merged = MCPUtility.mergeGeneratedTools( + MCPUtility.readMcpJson(outputFileLoc), generated, AUTOMATION_MCP_GENERATOR_ID, true); + + writeMcpJson(outputFileLoc, merged); + + MCPUtility.addMCPTag(project); + commitAndPush(project, projectId, assetsFolder, user); + } catch (Exception e) { + classLogger.warn("Failed to sync TriggerAutomation MCP tool for project {}: {}", projectId, e.getMessage(), e); + } + } + + // -- Private helpers ------------------------------------------------------------- + + private static JSONObject buildTriggerAutomationTool() { + JSONObject tool = new JSONObject(); + tool.put("name", "TriggerAutomation"); + tool.put("title", "Trigger Automation"); + tool.put("description", + "Manually triggers the automation configured for this project/app and returns a " + + "per-workflow summary once complete (e.g. \"Indexed 20 files\")."); + + JSONObject projectProp = new JSONObject(); + projectProp.put("type", "string"); + projectProp.put("title", "Project"); + projectProp.put("description", "The project (app) ID or alias to run the automation for."); + JSONObject properties = new JSONObject(); + properties.put(ReactorKeysEnum.PROJECT.getKey(), projectProp); + + JSONObject inputSchema = new JSONObject(); + inputSchema.put("type", "object"); + inputSchema.put("title", "TriggerAutomation_Arguments"); + inputSchema.put("properties", properties); + inputSchema.put("required", new JSONArray().put(ReactorKeysEnum.PROJECT.getKey())); + tool.put("inputSchema", inputSchema); + + JSONObject uiJson = new JSONObject(); + uiJson.put(MCPUtility.UI_DISPLAY_LOCATION, MCPDisplayOption.SIDEBAR.getValue()); + uiJson.put(MCPUtility.UI_RESOURCE_URI, "system://automation-workspace/"); + + JSONObject meta = new JSONObject(); + meta.put(MCPUtility.SMSS_FUNCTION_NAME, "TriggerAutomation"); + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPExecution.ASK.getValue()); + meta.put(MCPUtility.SMSS_MCP_UI, uiJson); + tool.put("_meta", meta); + + return tool; + } + + private static void writeMcpJson(String outputFileLoc, JSONArray tools) throws IOException { + JSONObject mcpJson = new JSONObject(); + mcpJson.put("tools", tools); + JSONObject fileMeta = new JSONObject(); + fileMeta.put("last_modified_date", LocalDate.now(ZoneOffset.UTC).format(DateTimeFormatter.ISO_LOCAL_DATE)); + mcpJson.put("_meta", fileMeta); + + File outputFile = new File(outputFileLoc); + outputFile.getParentFile().mkdirs(); + Files.writeString(outputFile.toPath(), mcpJson.toString(4), StandardCharsets.UTF_8); + } + + private static void commitAndPush(IProject project, String projectId, String assetsFolder, User user) { + String versionFolder = AssetUtility.getProjectVersionFolder(project.getProjectName(), projectId); + List gitRelativeFilePaths = new ArrayList<>(); + gitRelativeFilePaths.add(Constants.ASSETS_FOLDER + "/mcp/pixel_mcp.json"); + GitRepoUtils.addSpecificFiles(versionFolder, gitRelativeFilePaths); + GitRepoUtils.commitAddedFiles(versionFolder, "sync: automation MCP tool", user); + + if (ClusterUtil.IS_CLUSTER) { + ClusterUtil.pushProjectFolder(project, assetsFolder); + } + } +} diff --git a/src/prerna/reactor/automation/AutomationRunEngine.java b/src/prerna/reactor/automation/AutomationRunEngine.java index 85d649c875c..4ff736cf693 100644 --- a/src/prerna/reactor/automation/AutomationRunEngine.java +++ b/src/prerna/reactor/automation/AutomationRunEngine.java @@ -32,6 +32,7 @@ import java.time.LocalDateTime; import java.time.ZoneOffset; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -44,9 +45,11 @@ import org.apache.logging.log4j.Logger; import prerna.om.Insight; +import prerna.om.ThreadStore; import prerna.reactor.automation.nodes.AutomationNodeContext; import prerna.reactor.automation.nodes.AutomationNodeExecutors; import prerna.reactor.automation.nodes.IAutomationNodeExecutor; +import prerna.sablecc2.comm.PixelJobManager; import prerna.util.Utility; /** @@ -86,14 +89,24 @@ public static boolean requestCancellation(String runId) { * @param ordered nodes in execution order * @param configMap project automation config key-value pairs * @param insight the caller's insight context (propagated to each node executor) + * @return the run's final variable scope (trigger vars + every node's {@code outputVar} + * that completed successfully) - used by the caller to resolve a per-workflow + * summary message once the run finishes */ - public static void run(String runId, String projectId, + public static Map run(String runId, String projectId, List> ordered, Map configMap, Insight insight) { AtomicBoolean cancelled = new AtomicBoolean(false); CANCELLATION_FLAGS.put(runId, cancelled); ScheduledExecutorService heartbeat = startHeartbeat(runId); + // Captured once here - TriggerAutomationReactor runs on the virtual thread the platform's + // runPixelAsync endpoint spawns, so ThreadStore carries that job's id for the whole call. + // Used to stream per-node progress the same way HarnessToolExecutor streams tool-call + // progress during an agent turn (see PixelJobManager#addStreamOut), so the FE can poll + // getPixelJobStreaming(jobId) for live node status instead of inferring it from DB polls. + String jobId = ThreadStore.getJobId(); + Map scope = AutomationExecutionUtils.buildInitialScope(runId, insight.getUser()); int completedCount = 0; @@ -108,9 +121,11 @@ public static void run(String runId, String projectId, classLogger.info("Automation run {} cancelled before node {} ({})", runId, nodeId, nodeLabel); AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_CANCELLED, nodeId, "Run cancelled by user"); - return; + return scope; } + publishNodeEvent(jobId, nodeId, nodeLabel, AutomationConstants.NODE_STATUS_RUNNING, null, null, null); + Map nodeResult; try { nodeResult = executeSingleNode(runId, projectId, node, scope, configMap, cancelled, insight); @@ -118,10 +133,16 @@ public static void run(String runId, String projectId, classLogger.info("Automation run {} cancelled during node {} ({})", runId, nodeId, nodeLabel); AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_CANCELLED, nodeId, ace.getMessage()); - return; + publishNodeEvent(jobId, nodeId, nodeLabel, AutomationConstants.STATUS_CANCELLED, null, null, + ace.getMessage()); + return scope; } String status = (String) nodeResult.get(AutomationConstants.STATUS); + Object durationMs = nodeResult.get(AutomationConstants.DURATION_MS); + String preview = (String) nodeResult.get(AutomationConstants.OUTPUT_PREVIEW); + String errorMsg = (String) nodeResult.get(AutomationConstants.ERROR_MESSAGE); + publishNodeEvent(jobId, nodeId, nodeLabel, status, durationMs, preview, errorMsg); if (AutomationConstants.NODE_STATUS_SUCCESS.equals(status)) { if (outputVar != null && !outputVar.isEmpty() @@ -132,16 +153,16 @@ public static void run(String runId, String projectId, completedCount++; AutomationDatabaseUtility.updateHeartbeat(runId, completedCount); } else { - String errorMsg = (String) nodeResult.get(AutomationConstants.ERROR_MESSAGE); classLogger.warn("Automation run {} failed at node {} ({}): {}", runId, nodeId, nodeLabel, errorMsg); AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_FAILED, nodeId, errorMsg); - return; + return scope; } } classLogger.info("Automation run {} completed successfully ({}/{} nodes)", runId, completedCount, ordered.size()); AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_SUCCESS, null, null); + return scope; } finally { heartbeat.shutdownNow(); @@ -150,6 +171,42 @@ public static void run(String runId, String projectId, } } + // -- Streaming ------------------------------------------------------------------- + + /** + * Publishes a per-node progress event onto the pixel job's stream, mirroring + * {@code HarnessToolExecutor.publishToolResult} - the FE polls {@code getPixelJobStreaming(jobId)} + * (the same mechanism playground uses for live tool-call progress) to render each node's + * running/success/failed transition as it happens, instead of inferring progress from DB polls. + * A no-op when {@code jobId} is blank (e.g. called outside a {@code runPixelAsync} job). + */ + private static void publishNodeEvent(String jobId, String nodeId, String nodeLabel, String status, + Object durationMs, String preview, String errorMessage) { + if (jobId == null || jobId.isBlank()) { + return; + } + Map data = new LinkedHashMap<>(); + data.put("kind", "node-status"); + data.put(AutomationConstants.NODE_ID, nodeId); + data.put(AutomationConstants.NODE_LABEL, nodeLabel); + data.put(AutomationConstants.STATUS, status); + if (durationMs != null) { + data.put(AutomationConstants.DURATION_MS, durationMs); + } + if (preview != null) { + data.put(AutomationConstants.OUTPUT_PREVIEW, preview); + } + if (errorMessage != null) { + data.put(AutomationConstants.ERROR_MESSAGE, errorMessage); + } + data.put("timestamp", Instant.now().toString()); + + Map envelope = new LinkedHashMap<>(); + envelope.put("stream_type", "automation"); + envelope.put("data", data); + PixelJobManager.getManager().addStreamOut(jobId, envelope); + } + // -- Node execution ------------------------------------------------------------ private static Map executeSingleNode(String runId, String projectId, diff --git a/src/prerna/reactor/automation/SaveAutomationReactor.java b/src/prerna/reactor/automation/SaveAutomationReactor.java index e965a2a1f82..e3f6f71bb66 100644 --- a/src/prerna/reactor/automation/SaveAutomationReactor.java +++ b/src/prerna/reactor/automation/SaveAutomationReactor.java @@ -117,6 +117,11 @@ public NounMetadata execute() { } SecurityProjectUtils.updateProjectLastEditedDate(projectId); + + // Keep this project's own MCP tool catalog in sync with every save, so the automation is + // always discoverable as a "TriggerAutomation" tool without a separate manual step. + AutomationMcpSync.syncTriggerAutomationTool(project, projectId, this.insight.getUser()); + return new NounMetadata(true, PixelDataType.BOOLEAN, PixelOperationType.OPERATION); } diff --git a/src/prerna/reactor/automation/TriggerAutomationReactor.java b/src/prerna/reactor/automation/TriggerAutomationReactor.java index 1cf26ba463f..ea81a7d5b34 100644 --- a/src/prerna/reactor/automation/TriggerAutomationReactor.java +++ b/src/prerna/reactor/automation/TriggerAutomationReactor.java @@ -96,12 +96,13 @@ public NounMetadata execute() { classLogger.info("Automation run {} starting for project {}", runId, projectId); runStarted = true; - AutomationRunEngine.run(runId, projectId, ordered, configMap, this.insight); + Map finalScope = AutomationRunEngine.run(runId, projectId, ordered, configMap, this.insight); // Build final result in the same shape as GetAutomationRunReactor Map runDetail = AutomationDatabaseUtility.getRunDetail(runId); List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(runId); List> nodeResults = new ArrayList<>(); + int completedCount = 0; if (nodeOutputs != null) { for (Map output : nodeOutputs) { Map nodeResult = new HashMap<>(); @@ -112,6 +113,9 @@ public NounMetadata execute() { nodeResult.put(AutomationConstants.OUTPUT_PREVIEW, output.get(AutomationConstants.OUTPUT_PREVIEW)); nodeResult.put(AutomationConstants.ERROR_MESSAGE, output.get(AutomationConstants.ERROR_MESSAGE)); nodeResults.add(nodeResult); + if (AutomationConstants.NODE_STATUS_SUCCESS.equals(output.get(AutomationConstants.STATUS))) { + completedCount++; + } } } if (runDetail == null) { @@ -120,6 +124,15 @@ public NounMetadata execute() { runDetail.put(AutomationConstants.PROJECT_ID, projectId); } runDetail.put(AutomationConstants.RESULT_NODE_RESULTS, nodeResults); + + // Per-workflow human-readable summary (e.g. "Indexed 20 files") instead of raw JSON, + // surfaced as the primary MCP/agent-visible result. + boolean runSucceeded = AutomationConstants.STATUS_SUCCESS.equals(runDetail.get(AutomationConstants.STATUS)); + String summary = runSucceeded + ? AutomationExecutionUtils.buildSummaryMessage(doc, finalScope, configMap, completedCount, ordered.size()) + : buildFailureSummary(runDetail); + runDetail.put(AutomationConstants.RESULT_SUMMARY, summary); + return new NounMetadata(runDetail, PixelDataType.MAP, PixelOperationType.OPERATION); } catch (Exception e) { @@ -136,6 +149,14 @@ public NounMetadata execute() { // -- Helpers ------------------------------------------------------------------- + /** Null-safe failure summary - {@code FAILED_NODE_ID}/{@code ERROR_MESSAGE} may be absent. */ + private static String buildFailureSummary(Map runDetail) { + Object failedNodeId = runDetail.get(AutomationConstants.FAILED_NODE_ID); + Object errorMessage = runDetail.get(AutomationConstants.ERROR_MESSAGE); + return "Automation failed at node " + (failedNodeId != null ? failedNodeId : "unknown") + + ": " + (errorMessage != null ? errorMessage : "no error details available"); + } + private String getProjectId() { String projectId = this.keyValue.get(this.keysToGet[0]); if (projectId == null || projectId.isEmpty()) { @@ -160,6 +181,11 @@ public Map getMcpToolMetadata() { Map meta = new HashMap<>(); meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPExecution.ASK.getValue()); meta.put(MCPUtility.UI_DISPLAY_LOCATION, MCPDisplayOption.SIDEBAR.getValue()); + // Same "system app" resourceURI scheme used by the Playwright browser-sockets tool + // (see PlaywrightMCPToolBuilder) - resolved client-side to the automation workspace's + // own build output (../../automation-workspace/dist/) so the exact same UI renders + // whether embedded directly in the client app or iframed as an MCP sidebar tool. + meta.put(MCPUtility.UI_RESOURCE_URI, "system://automation-workspace/"); return meta; } From 56c5ee01d27ff83ec8f6c17bdd949fd05a8111f7 Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Thu, 30 Jul 2026 17:30:38 -0400 Subject: [PATCH 13/25] feat: add mcp tool completion, node context, and bug fixes --- .../automation/AutomationConstants.java | 2 ++ .../reactor/automation/AutomationMcpSync.java | 9 +++--- .../SaveAutomationConfigReactor.java | 4 +-- .../automation/SaveAutomationReactor.java | 4 +-- .../automation/TriggerAutomationReactor.java | 29 ++++++++++++++++--- 5 files changed, 36 insertions(+), 12 deletions(-) diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index 66981f36afd..f4de3068c44 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -217,6 +217,8 @@ private AutomationConstants() {} public static final String RESULT_OUTPUT_VALUE = "outputValue"; /** Human-readable, per-workflow summary surfaced to MCP/agent consumers (see {@link #DOC_RESULT_MESSAGE_TEMPLATE}). */ public static final String RESULT_SUMMARY = "summary"; + /** Enriched summary including per-step output previews, sent as the MCP tool response so the LLM can describe what happened. */ + public static final String RESULT_LLM_CONTEXT = "llmContext"; // -- Pixel execution defaults ---------------------------------------------------- diff --git a/src/prerna/reactor/automation/AutomationMcpSync.java b/src/prerna/reactor/automation/AutomationMcpSync.java index e76e6977cd0..ab7a15967f7 100644 --- a/src/prerna/reactor/automation/AutomationMcpSync.java +++ b/src/prerna/reactor/automation/AutomationMcpSync.java @@ -100,7 +100,7 @@ public static void syncTriggerAutomationTool(IProject project, String projectId, } try { - JSONArray generated = new JSONArray().put(buildTriggerAutomationTool()); + JSONArray generated = new JSONArray().put(buildTriggerAutomationTool(projectId)); MCPUtility.stampGenerator(generated, AUTOMATION_MCP_GENERATOR_ID); String assetsFolder = AssetUtility.getProjectAssetsFolder(projectId); @@ -119,7 +119,7 @@ public static void syncTriggerAutomationTool(IProject project, String projectId, // -- Private helpers ------------------------------------------------------------- - private static JSONObject buildTriggerAutomationTool() { + private static JSONObject buildTriggerAutomationTool(String projectId) { JSONObject tool = new JSONObject(); tool.put("name", "TriggerAutomation"); tool.put("title", "Trigger Automation"); @@ -130,7 +130,8 @@ private static JSONObject buildTriggerAutomationTool() { JSONObject projectProp = new JSONObject(); projectProp.put("type", "string"); projectProp.put("title", "Project"); - projectProp.put("description", "The project (app) ID or alias to run the automation for."); + projectProp.put("description", "The project ID for this automation. Always use: " + projectId); + projectProp.put("default", projectId); JSONObject properties = new JSONObject(); properties.put(ReactorKeysEnum.PROJECT.getKey(), projectProp); @@ -143,7 +144,7 @@ private static JSONObject buildTriggerAutomationTool() { JSONObject uiJson = new JSONObject(); uiJson.put(MCPUtility.UI_DISPLAY_LOCATION, MCPDisplayOption.SIDEBAR.getValue()); - uiJson.put(MCPUtility.UI_RESOURCE_URI, "system://automation-workspace/"); + uiJson.put(MCPUtility.UI_RESOURCE_URI, "system://automation-workspace/?readOnly=1"); JSONObject meta = new JSONObject(); meta.put(MCPUtility.SMSS_FUNCTION_NAME, "TriggerAutomation"); diff --git a/src/prerna/reactor/automation/SaveAutomationConfigReactor.java b/src/prerna/reactor/automation/SaveAutomationConfigReactor.java index e4f2e0b9a64..4029be39ffb 100644 --- a/src/prerna/reactor/automation/SaveAutomationConfigReactor.java +++ b/src/prerna/reactor/automation/SaveAutomationConfigReactor.java @@ -29,7 +29,7 @@ import java.io.File; import java.io.IOException; -import java.net.URLDecoder; +import java.util.Base64; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.HashMap; @@ -75,7 +75,7 @@ public NounMetadata execute() { String config; try { - config = URLDecoder.decode(configEncoded != null ? configEncoded : AutomationConstants.EMPTY_JSON_ARRAY, StandardCharsets.UTF_8); + config = new String(Base64.getDecoder().decode(configEncoded != null ? configEncoded : AutomationConstants.EMPTY_JSON_ARRAY), StandardCharsets.UTF_8); } catch (Exception e) { config = configEncoded != null ? configEncoded : AutomationConstants.EMPTY_JSON_ARRAY; } diff --git a/src/prerna/reactor/automation/SaveAutomationReactor.java b/src/prerna/reactor/automation/SaveAutomationReactor.java index e3f6f71bb66..4214a3256e4 100644 --- a/src/prerna/reactor/automation/SaveAutomationReactor.java +++ b/src/prerna/reactor/automation/SaveAutomationReactor.java @@ -29,7 +29,7 @@ import java.io.File; import java.io.IOException; -import java.net.URLDecoder; +import java.util.Base64; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.util.ArrayList; @@ -81,7 +81,7 @@ public NounMetadata execute() { String json; try { - json = URLDecoder.decode(jsonEncoded, StandardCharsets.UTF_8); + json = new String(Base64.getDecoder().decode(jsonEncoded), StandardCharsets.UTF_8); } catch (Exception e) { json = jsonEncoded; } diff --git a/src/prerna/reactor/automation/TriggerAutomationReactor.java b/src/prerna/reactor/automation/TriggerAutomationReactor.java index ea81a7d5b34..9111441cea4 100644 --- a/src/prerna/reactor/automation/TriggerAutomationReactor.java +++ b/src/prerna/reactor/automation/TriggerAutomationReactor.java @@ -111,6 +111,7 @@ public NounMetadata execute() { nodeResult.put(AutomationConstants.STATUS, output.get(AutomationConstants.STATUS)); nodeResult.put(AutomationConstants.DURATION_MS, output.get(AutomationConstants.DURATION_MS)); nodeResult.put(AutomationConstants.OUTPUT_PREVIEW, output.get(AutomationConstants.OUTPUT_PREVIEW)); + nodeResult.put(AutomationConstants.OUTPUT_VALUE, output.get(AutomationConstants.OUTPUT_VALUE)); nodeResult.put(AutomationConstants.ERROR_MESSAGE, output.get(AutomationConstants.ERROR_MESSAGE)); nodeResults.add(nodeResult); if (AutomationConstants.NODE_STATUS_SUCCESS.equals(output.get(AutomationConstants.STATUS))) { @@ -118,6 +119,13 @@ public NounMetadata execute() { } } } + // Trigger nodes succeed immediately in the engine but never write a SUCCESS + // DB record, so add them back so the count reflects what the user sees. + int triggerCount = (int) ordered.stream() + .filter(n -> AutomationConstants.NODE_TRIGGER.equals(n.get(AutomationConstants.NODE_FIELD_TYPE))) + .count(); + completedCount += triggerCount; + if (runDetail == null) { runDetail = new HashMap<>(); runDetail.put(AutomationConstants.RUN_ID, runId); @@ -125,14 +133,27 @@ public NounMetadata execute() { } runDetail.put(AutomationConstants.RESULT_NODE_RESULTS, nodeResults); - // Per-workflow human-readable summary (e.g. "Indexed 20 files") instead of raw JSON, - // surfaced as the primary MCP/agent-visible result. boolean runSucceeded = AutomationConstants.STATUS_SUCCESS.equals(runDetail.get(AutomationConstants.STATUS)); + // Short summary shown in the sidebar UI. String summary = runSucceeded ? AutomationExecutionUtils.buildSummaryMessage(doc, finalScope, configMap, completedCount, ordered.size()) : buildFailureSummary(runDetail); runDetail.put(AutomationConstants.RESULT_SUMMARY, summary); + // Enriched context sent to the LLM as the MCP tool response, so it can describe + // what each step actually did rather than just echoing the node count. + if (runSucceeded) { + StringBuilder llmContext = new StringBuilder(summary); + for (Map step : nodeResults) { + String label = (String) step.get(AutomationConstants.NODE_LABEL); + String preview = (String) step.get(AutomationConstants.OUTPUT_PREVIEW); + if (preview != null && !preview.isBlank()) { + llmContext.append("\n- ").append(label).append(": ").append(preview); + } + } + runDetail.put(AutomationConstants.RESULT_LLM_CONTEXT, llmContext.toString()); + } + return new NounMetadata(runDetail, PixelDataType.MAP, PixelOperationType.OPERATION); } catch (Exception e) { @@ -185,13 +206,13 @@ public Map getMcpToolMetadata() { // (see PlaywrightMCPToolBuilder) - resolved client-side to the automation workspace's // own build output (../../automation-workspace/dist/) so the exact same UI renders // whether embedded directly in the client app or iframed as an MCP sidebar tool. - meta.put(MCPUtility.UI_RESOURCE_URI, "system://automation-workspace/"); + meta.put(MCPUtility.UI_RESOURCE_URI, "system://automation-workspace/?readOnly=1"); return meta; } @Override public String getReactorDescription() { - return "Manually triggers an automation run for the given project and returns a run ID for polling."; + return "Manually triggers an automation run for the given project and returns a per-step summary once complete."; } @Override From a2c4e69e308f0534c1d4fef92a1c8deb47646dc8 Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Tue, 4 Aug 2026 16:32:36 -0400 Subject: [PATCH 14/25] feat: add playground inputs to automation nodes --- .../pipeline/PipelineInvocationHandler.java | 7 +- .../automation/AutomationConstants.java | 1 + .../automation/AutomationExecutionUtils.java | 91 ++++++++- .../reactor/automation/AutomationMcpSync.java | 136 +++++++++++++- .../GetAutomationSchemaReactor.java | 176 ++++++++++++++++++ .../automation/SaveAutomationReactor.java | 2 +- .../automation/TriggerAutomationReactor.java | 9 +- 7 files changed, 410 insertions(+), 12 deletions(-) create mode 100644 src/prerna/reactor/automation/GetAutomationSchemaReactor.java diff --git a/src/prerna/engine/impl/pipeline/PipelineInvocationHandler.java b/src/prerna/engine/impl/pipeline/PipelineInvocationHandler.java index fb07d15558d..bce66f7454d 100644 --- a/src/prerna/engine/impl/pipeline/PipelineInvocationHandler.java +++ b/src/prerna/engine/impl/pipeline/PipelineInvocationHandler.java @@ -61,6 +61,8 @@ import com.github.f4b6a3.uuid.alt.GUID; import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializer; import com.google.gson.ToNumberPolicy; import prerna.engine.api.IEngine; @@ -114,7 +116,10 @@ public class PipelineInvocationHandler implements InvocationHandler { .registerTypeAdapter(ZoneOffset.class, new ZoneOffsetTypeAdapter()) .registerTypeAdapter(Insight.class, new LoggingInsightAdapter()) .registerTypeAdapter(LocalDateTime.class, new LocalDateTimeAdapter()) - .registerTypeAdapter(ZonedDateTime.class, new ZonedDateTimeAdapter()).create(); + .registerTypeAdapter(ZonedDateTime.class, new ZonedDateTimeAdapter()) + .registerTypeHierarchyAdapter(Throwable.class, + (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.toString())) + .create(); private final String REQUEST_NOT_TRACKED = "REQUEST NOT TRACKED"; private final String RESPONSE_NOT_TRACKED = "RESPONSE NOT TRACKED"; diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index f4de3068c44..4c08223776a 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -222,6 +222,7 @@ private AutomationConstants() {} // -- Pixel execution defaults ---------------------------------------------------- + public static final String AUTOMATION_INPUTS_KEY = "inputs"; public static final int DEFAULT_TIMEOUT_SECONDS = 300; // -- Data type constants (for table creation) ---------------------------------- diff --git a/src/prerna/reactor/automation/AutomationExecutionUtils.java b/src/prerna/reactor/automation/AutomationExecutionUtils.java index fa9d7930f21..1a6d6615886 100644 --- a/src/prerna/reactor/automation/AutomationExecutionUtils.java +++ b/src/prerna/reactor/automation/AutomationExecutionUtils.java @@ -31,6 +31,10 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -45,6 +49,8 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializer; import com.google.gson.reflect.TypeToken; import prerna.auth.User; @@ -68,7 +74,23 @@ public final class AutomationExecutionUtils { * {@code nodes} sub-package has one shared instance to reuse instead of each * file declaring its own. */ - public static final Gson GSON = new GsonBuilder().disableHtmlEscaping().create(); + public static final Gson GSON = new GsonBuilder() + .disableHtmlEscaping() + .registerTypeHierarchyAdapter(ZoneId.class, + (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.getId())) + .registerTypeAdapter(ZonedDateTime.class, + (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.toString())) + .registerTypeAdapter(OffsetDateTime.class, + (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.toString())) + .registerTypeAdapter(LocalDateTime.class, + (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.toString())) + .registerTypeAdapter(LocalDate.class, + (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.toString())) + .registerTypeAdapter(Instant.class, + (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.toString())) + .registerTypeHierarchyAdapter(Throwable.class, + (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.toString())) + .create(); private AutomationExecutionUtils() {} @@ -350,6 +372,73 @@ public static String generatePreview(String s) { ? s : s.substring(0, AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH); } + // -- Playground input helpers -------------------------------------------------- + + /** + * Extracts playground input values from the Pixel MAP noun and overwrites the matching + * config field on each node before execution. Nodes without a {@code playgroundFillable} + * list or whose listed fields are absent from the node config are skipped with a warning. + * + * @param nodes ordered node list from the automation document (mutated in place) + * @param inputsMap raw MAP noun value from the {@code inputs} Pixel parameter, or {@code null} + * @return the flattened {@code String → String} inputs map (empty when no inputs were passed) + */ + @SuppressWarnings("unchecked") + public static Map applyPlaygroundInputs(List> nodes, Map inputsMap) { + Map playgroundInputs = new HashMap<>(); + if (inputsMap != null) { + for (Map.Entry entry : inputsMap.entrySet()) { + if (entry.getValue() != null) { + playgroundInputs.put(entry.getKey(), entry.getValue().toString()); + } + } + } + if (!playgroundInputs.isEmpty()) { + for (Map node : nodes) { + List fillable = (List) node.get("playgroundFillable"); + if (fillable == null || fillable.isEmpty()) continue; + String nodeLabel = (String) node.get(AutomationConstants.NODE_FIELD_LABEL); + Map config = (Map) node.get(AutomationConstants.NODE_FIELD_CONFIG); + if (config == null) continue; + for (String fieldName : fillable) { + String paramName = buildPlaygroundParamName(nodeLabel, fieldName); + String value = playgroundInputs.get(paramName); + if (value != null) { + if (config.containsKey(fieldName)) { + config.put(fieldName, value); + } else { + classLogger.warn("Playground input '{}' targets field '{}' which does not exist in node '{}' config - skipping", + paramName, fieldName, nodeLabel); + } + } + } + } + } + return playgroundInputs; + } + + /** + * Builds a stable MCP parameter name for a playground-fillable node field. + * Slugifies the node label (lowercase, non-alphanumeric chars → underscore, collapsed) + * and appends the field name. Falls back to {@code "input_" + fieldName} if label is blank. + * + * @param nodeLabel the node's display label (may be null or empty) + * @param fieldName the config field name (e.g. "expression", "command") + * @return a deterministic, URL-safe parameter name + */ + public static String buildPlaygroundParamName(String nodeLabel, String fieldName) { + if (nodeLabel != null && !nodeLabel.trim().isEmpty()) { + String slug = nodeLabel.trim() + .toLowerCase() + .replaceAll("[^a-z0-9]+", "_") + .replaceAll("^_+|_+$", ""); + if (!slug.isEmpty()) { + return slug + "_" + fieldName; + } + } + return "input_" + fieldName; + } + // -- Config value coercion ----------------------------------------------------- /** diff --git a/src/prerna/reactor/automation/AutomationMcpSync.java b/src/prerna/reactor/automation/AutomationMcpSync.java index ab7a15967f7..fdfaf1afd0d 100644 --- a/src/prerna/reactor/automation/AutomationMcpSync.java +++ b/src/prerna/reactor/automation/AutomationMcpSync.java @@ -92,7 +92,7 @@ private AutomationMcpSync() { * @param projectId the project id (used even when {@code project} is present, for clarity) * @param user the user performing the save, used as the git commit author */ - public static void syncTriggerAutomationTool(IProject project, String projectId, User user) { + public static void syncTriggerAutomationTool(IProject project, String projectId, User user, String automationJson) { if (project == null) { classLogger.warn("Skipping automation MCP tool sync for project {}: project could not be loaded.", projectId); @@ -100,7 +100,11 @@ public static void syncTriggerAutomationTool(IProject project, String projectId, } try { - JSONArray generated = new JSONArray().put(buildTriggerAutomationTool(projectId)); + boolean hasDbNodes = hasPlaygroundDbNodes(automationJson); + JSONArray generated = new JSONArray().put(buildTriggerAutomationTool(projectId, automationJson, hasDbNodes)); + if (hasDbNodes) { + generated.put(buildGetAutomationSchemaTool(projectId)); + } MCPUtility.stampGenerator(generated, AUTOMATION_MCP_GENERATOR_ID); String assetsFolder = AssetUtility.getProjectAssetsFolder(projectId); @@ -113,19 +117,23 @@ public static void syncTriggerAutomationTool(IProject project, String projectId, MCPUtility.addMCPTag(project); commitAndPush(project, projectId, assetsFolder, user); } catch (Exception e) { - classLogger.warn("Failed to sync TriggerAutomation MCP tool for project {}: {}", projectId, e.getMessage(), e); + classLogger.warn("Failed to sync TriggerAutomation MCP tool for project {}", projectId, e); } } // -- Private helpers ------------------------------------------------------------- - private static JSONObject buildTriggerAutomationTool(String projectId) { + private static JSONObject buildTriggerAutomationTool(String projectId, String automationJson, boolean hasDbNodes) { JSONObject tool = new JSONObject(); tool.put("name", "TriggerAutomation"); tool.put("title", "Trigger Automation"); - tool.put("description", - "Manually triggers the automation configured for this project/app and returns a " - + "per-workflow summary once complete (e.g. \"Indexed 20 files\")."); + String description = "Manually triggers the automation configured for this project/app and returns a " + + "per-workflow summary once complete (e.g. \"Indexed 20 files\")."; + if (hasDbNodes) { + description += " This automation has database nodes that accept SQL queries — call GetAutomationSchema first" + + " to discover the exact table and column names before writing SQL."; + } + tool.put("description", description); JSONObject projectProp = new JSONObject(); projectProp.put("type", "string"); @@ -135,6 +143,45 @@ private static JSONObject buildTriggerAutomationTool(String projectId) { JSONObject properties = new JSONObject(); properties.put(ReactorKeysEnum.PROJECT.getKey(), projectProp); + JSONObject inputsProperties = new JSONObject(); + try { + if (automationJson != null && !automationJson.isBlank()) { + JSONObject doc = new JSONObject(automationJson); + JSONObject graph = doc.optJSONObject("graph"); + JSONArray nodes = graph != null ? graph.optJSONArray("nodes") : null; + if (nodes != null) { + for (int i = 0; i < nodes.length(); i++) { + JSONObject node = nodes.optJSONObject(i); + if (node == null) continue; + String nodeLabel = node.optString("label", ""); + JSONArray fillable = node.optJSONArray("playgroundFillable"); + if (fillable == null || fillable.length() == 0) continue; + String nodeType = node.optString("type", ""); + for (int j = 0; j < fillable.length(); j++) { + String fieldName = fillable.optString(j); + if (fieldName == null || fieldName.isBlank()) continue; + String paramName = AutomationExecutionUtils.buildPlaygroundParamName(nodeLabel, fieldName); + String paramDescription = buildPlaygroundParamDescription(nodeType, fieldName); + JSONObject paramProp = new JSONObject(); + paramProp.put("type", "string"); + paramProp.put("description", paramDescription); + inputsProperties.put(paramName, paramProp); + } + } + } + } + } catch (Exception e) { + classLogger.warn("Failed to scan automation nodes for playground inputs for project {}", projectId, e); + } + + if (!inputsProperties.isEmpty()) { + JSONObject inputsProp = new JSONObject(); + inputsProp.put("type", "object"); + inputsProp.put("description", "Optional inputs to inject into automation nodes before running. Populate fields with values relevant to the user's request."); + inputsProp.put("properties", inputsProperties); + properties.put(AutomationConstants.AUTOMATION_INPUTS_KEY, inputsProp); + } + JSONObject inputSchema = new JSONObject(); inputSchema.put("type", "object"); inputSchema.put("title", "TriggerAutomation_Arguments"); @@ -155,6 +202,62 @@ private static JSONObject buildTriggerAutomationTool(String projectId) { return tool; } + /** Returns true if any database-engine node has {@code expression} in its {@code playgroundFillable} list. */ + private static boolean hasPlaygroundDbNodes(String automationJson) { + if (automationJson == null || automationJson.isBlank()) return false; + try { + JSONObject doc = new JSONObject(automationJson); + JSONObject graph = doc.optJSONObject("graph"); + JSONArray nodes = graph != null ? graph.optJSONArray("nodes") : null; + if (nodes == null) return false; + for (int i = 0; i < nodes.length(); i++) { + JSONObject node = nodes.optJSONObject(i); + if (node == null) continue; + if (!AutomationConstants.NODE_DATABASE_ENGINE.equals(node.optString("type"))) continue; + JSONArray fillable = node.optJSONArray("playgroundFillable"); + if (fillable == null) continue; + for (int j = 0; j < fillable.length(); j++) { + if (AutomationConstants.CONFIG_EXPRESSION.equals(fillable.optString(j))) return true; + } + } + } catch (Exception e) { + classLogger.warn("Failed to parse automation JSON while checking for DB nodes", e); + } + return false; + } + + /** Builds the auto-executable {@code GetAutomationSchema} companion tool. */ + private static JSONObject buildGetAutomationSchemaTool(String projectId) { + JSONObject tool = new JSONObject(); + tool.put("name", "GetAutomationSchema"); + tool.put("title", "Get Automation Database Schema"); + tool.put("description", + "Returns the physical table and column names for database nodes in this automation that accept SQL input. " + + "Call this before TriggerAutomation when you need to write a SQL query — it gives you the exact " + + "table and column names available in the database."); + + JSONObject projectProp = new JSONObject(); + projectProp.put("type", "string"); + projectProp.put("description", "The project ID for this automation. Always use: " + projectId); + projectProp.put("default", projectId); + JSONObject properties = new JSONObject(); + properties.put(ReactorKeysEnum.PROJECT.getKey(), projectProp); + + JSONObject inputSchema = new JSONObject(); + inputSchema.put("type", "object"); + inputSchema.put("title", "GetAutomationSchema_Arguments"); + inputSchema.put("properties", properties); + inputSchema.put("required", new JSONArray().put(ReactorKeysEnum.PROJECT.getKey())); + tool.put("inputSchema", inputSchema); + + JSONObject meta = new JSONObject(); + meta.put(MCPUtility.SMSS_FUNCTION_NAME, "GetAutomationSchema"); + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPExecution.AUTO.getValue()); + tool.put("_meta", meta); + + return tool; + } + private static void writeMcpJson(String outputFileLoc, JSONArray tools) throws IOException { JSONObject mcpJson = new JSONObject(); mcpJson.put("tools", tools); @@ -167,6 +270,25 @@ private static void writeMcpJson(String outputFileLoc, JSONArray tools) throws I Files.writeString(outputFile.toPath(), mcpJson.toString(4), StandardCharsets.UTF_8); } + private static String buildPlaygroundParamDescription(String nodeType, String fieldName) { + if ("database-engine".equals(nodeType) && "expression".equals(fieldName)) { + return "SQL query to execute against the connected database"; + } + if ("model-engine".equals(nodeType) && "command".equals(fieldName)) { + return "Natural language prompt to send to the language model"; + } + if ("model-engine".equals(nodeType) && "context".equals(fieldName)) { + return "System instructions for the language model's behavior"; + } + if ("vector-engine".equals(nodeType) && "command".equals(fieldName)) { + return "Search query to run against the vector database"; + } + if ("function-engine".equals(nodeType) && "params".equals(fieldName)) { + return "JSON parameters to pass to the function"; + } + return "Input for the " + fieldName + " field"; + } + private static void commitAndPush(IProject project, String projectId, String assetsFolder, User user) { String versionFolder = AssetUtility.getProjectVersionFolder(project.getProjectName(), projectId); List gitRelativeFilePaths = new ArrayList<>(); diff --git a/src/prerna/reactor/automation/GetAutomationSchemaReactor.java b/src/prerna/reactor/automation/GetAutomationSchemaReactor.java new file mode 100644 index 00000000000..fc65f963570 --- /dev/null +++ b/src/prerna/reactor/automation/GetAutomationSchemaReactor.java @@ -0,0 +1,176 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.engine.api.IDatabaseEngine; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Returns the physical table and column names for every database-engine node + * in the automation that exposes its SQL expression as a playground-fillable + * field. Used by the LLM to discover the schema before writing SQL queries for + * {@code TriggerAutomation}. + * + *

Pixel: {@code GetAutomationSchema(project=["appId"])} + */ +public class GetAutomationSchemaReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GetAutomationSchemaReactor.class); + + public GetAutomationSchemaReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey() }; + this.keyRequired = new int[] { 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + if (projectId == null || projectId.isBlank()) { + throw new IllegalArgumentException("Must provide a project id"); + } + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + + Map doc = AutomationExecutionUtils.loadAutomationDoc(projectId); + @SuppressWarnings("unchecked") + Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); + @SuppressWarnings("unchecked") + List> nodes = graph != null ? (List>) graph.get(AutomationConstants.DOC_NODES) : null; + + List> result = new ArrayList<>(); + if (nodes != null) { + for (Map node : nodes) { + if (!AutomationConstants.NODE_DATABASE_ENGINE.equals(node.get(AutomationConstants.NODE_FIELD_TYPE))) { + continue; + } + @SuppressWarnings("unchecked") + List fillable = (List) node.get("playgroundFillable"); + if (fillable == null || !fillable.contains(AutomationConstants.CONFIG_EXPRESSION)) { + continue; + } + @SuppressWarnings("unchecked") + Map config = (Map) node.get(AutomationConstants.NODE_FIELD_CONFIG); + if (config == null) continue; + + String nodeLabel = (String) node.get(AutomationConstants.NODE_FIELD_LABEL); + Object engineIdObj = config.get(AutomationConstants.CONFIG_ENGINE_ID); + if (engineIdObj == null || engineIdObj.toString().isBlank()) continue; + + String engineId = engineIdObj.toString(); + Map nodeSchema = buildNodeSchema(nodeLabel, engineId); + if (nodeSchema != null) { + result.add(nodeSchema); + } + } + } + + Map output = new HashMap<>(); + output.put("nodes", result); + if (result.isEmpty()) { + output.put("message", "No database nodes with playground-fillable SQL expressions found in this automation."); + } + return new NounMetadata(output, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + private Map buildNodeSchema(String nodeLabel, String engineId) { + IDatabaseEngine engine; + try { + engine = Utility.getDatabase(engineId); + } catch (Exception e) { + classLogger.warn("Could not load database engine {} for schema discovery", engineId, e); + return null; + } + if (engine == null) { + classLogger.warn("Database engine {} not found for node '{}'", engineId, nodeLabel); + return null; + } + + List conceptUris = engine.getPhysicalConcepts(); + if (conceptUris == null || conceptUris.isEmpty()) { + return null; + } + + List> tables = new ArrayList<>(); + for (String conceptUri : conceptUris) { + String tableName = Utility.getInstanceName(conceptUri); + if (tableName == null || tableName.isBlank()) continue; + + List columns = new ArrayList<>(); + List propUris = engine.getPropertyUris4PhysicalUri(conceptUri); + if (propUris != null) { + for (String propUri : propUris) { + String colName = Utility.getInstanceName(propUri); + if (colName != null && !colName.isBlank()) { + columns.add(colName); + } + } + } + + Map tableEntry = new HashMap<>(); + tableEntry.put("name", tableName); + tableEntry.put("columns", columns); + tables.add(tableEntry); + } + + if (tables.isEmpty()) return null; + + Map nodeSchema = new HashMap<>(); + nodeSchema.put("nodeLabel", nodeLabel != null ? nodeLabel : AutomationConstants.UNNAMED_NODE_LABEL); + nodeSchema.put("engineId", engineId); + nodeSchema.put("tables", tables); + return nodeSchema; + } + + @Override + public String getReactorDescription() { + return "Returns the physical table and column names for database-engine nodes in the automation that accept SQL input. Call this before TriggerAutomation to discover the exact table and column names to use in your SQL query."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) return "The project (app) ID or alias to retrieve the database schema for."; + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/reactor/automation/SaveAutomationReactor.java b/src/prerna/reactor/automation/SaveAutomationReactor.java index 4214a3256e4..8516d646681 100644 --- a/src/prerna/reactor/automation/SaveAutomationReactor.java +++ b/src/prerna/reactor/automation/SaveAutomationReactor.java @@ -120,7 +120,7 @@ public NounMetadata execute() { // Keep this project's own MCP tool catalog in sync with every save, so the automation is // always discoverable as a "TriggerAutomation" tool without a separate manual step. - AutomationMcpSync.syncTriggerAutomationTool(project, projectId, this.insight.getUser()); + AutomationMcpSync.syncTriggerAutomationTool(project, projectId, this.insight.getUser(), json); return new NounMetadata(true, PixelDataType.BOOLEAN, PixelOperationType.OPERATION); } diff --git a/src/prerna/reactor/automation/TriggerAutomationReactor.java b/src/prerna/reactor/automation/TriggerAutomationReactor.java index 9111441cea4..5936d204aa2 100644 --- a/src/prerna/reactor/automation/TriggerAutomationReactor.java +++ b/src/prerna/reactor/automation/TriggerAutomationReactor.java @@ -58,8 +58,8 @@ public class TriggerAutomationReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(TriggerAutomationReactor.class); public TriggerAutomationReactor() { - this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey() }; - this.keyRequired = new int[] { 1 }; + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), AutomationConstants.AUTOMATION_INPUTS_KEY }; + this.keyRequired = new int[] { 1, 0 }; } @Override @@ -90,6 +90,10 @@ public NounMetadata execute() { throw new IllegalArgumentException("Automation has no nodes to execute"); } + @SuppressWarnings("unchecked") + Map inputsMap = this.getMap(AutomationConstants.AUTOMATION_INPUTS_KEY); + AutomationExecutionUtils.applyPlaygroundInputs(ordered, inputsMap); + AutomationDatabaseUtility.insertRun(runId, projectId, AutomationConstants.DEFAULT_AUTOMATION_ID, AutomationConstants.TRIGGER_MANUAL, ordered.size(), userId); AutomationDatabaseUtility.insertAllNodeOutputs(runId, ordered); @@ -218,6 +222,7 @@ public String getReactorDescription() { @Override protected String getDescriptionForKey(String key) { if (ReactorKeysEnum.PROJECT.getKey().equals(key)) return "The project (app) ID or alias to run the automation for."; + if (AutomationConstants.AUTOMATION_INPUTS_KEY.equals(key)) return "Optional map of playground-supplied values to inject into automation node fields before running. Keys are parameter names from the automation's MCP tool schema."; return super.getDescriptionForKey(key); } } From dcb3c5163c6051d233e17f3d3b43593d81212089 Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Wed, 5 Aug 2026 08:53:18 -0400 Subject: [PATCH 15/25] feat: add playground trigger type in logs --- .../reactor/automation/AutomationConstants.java | 2 ++ .../reactor/automation/AutomationMcpSync.java | 7 +++++++ .../automation/TriggerAutomationReactor.java | 15 ++++++++++++--- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index 4c08223776a..21abb3adc68 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -94,6 +94,7 @@ private AutomationConstants() {} // -- Trigger types ------------------------------------------------------------- public static final String TRIGGER_MANUAL = "MANUAL"; + public static final String TRIGGER_PLAYGROUND = "PLAYGROUND"; // -- Node types (Phase 1) ------------------------------------------------------ @@ -223,6 +224,7 @@ private AutomationConstants() {} // -- Pixel execution defaults ---------------------------------------------------- public static final String AUTOMATION_INPUTS_KEY = "inputs"; + public static final String AUTOMATION_TRIGGER_TYPE_KEY = "triggerType"; public static final int DEFAULT_TIMEOUT_SECONDS = 300; // -- Data type constants (for table creation) ---------------------------------- diff --git a/src/prerna/reactor/automation/AutomationMcpSync.java b/src/prerna/reactor/automation/AutomationMcpSync.java index fdfaf1afd0d..fe05505afe8 100644 --- a/src/prerna/reactor/automation/AutomationMcpSync.java +++ b/src/prerna/reactor/automation/AutomationMcpSync.java @@ -182,6 +182,13 @@ private static JSONObject buildTriggerAutomationTool(String projectId, String au properties.put(AutomationConstants.AUTOMATION_INPUTS_KEY, inputsProp); } + JSONObject triggerTypeProp = new JSONObject(); + triggerTypeProp.put("type", "string"); + triggerTypeProp.put("title", "Trigger Type"); + triggerTypeProp.put("description", "How this automation was triggered. Always use: " + AutomationConstants.TRIGGER_PLAYGROUND); + triggerTypeProp.put("default", AutomationConstants.TRIGGER_PLAYGROUND); + properties.put(AutomationConstants.AUTOMATION_TRIGGER_TYPE_KEY, triggerTypeProp); + JSONObject inputSchema = new JSONObject(); inputSchema.put("type", "object"); inputSchema.put("title", "TriggerAutomation_Arguments"); diff --git a/src/prerna/reactor/automation/TriggerAutomationReactor.java b/src/prerna/reactor/automation/TriggerAutomationReactor.java index 5936d204aa2..52a69469e7b 100644 --- a/src/prerna/reactor/automation/TriggerAutomationReactor.java +++ b/src/prerna/reactor/automation/TriggerAutomationReactor.java @@ -58,8 +58,8 @@ public class TriggerAutomationReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(TriggerAutomationReactor.class); public TriggerAutomationReactor() { - this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), AutomationConstants.AUTOMATION_INPUTS_KEY }; - this.keyRequired = new int[] { 1, 0 }; + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), AutomationConstants.AUTOMATION_INPUTS_KEY, AutomationConstants.AUTOMATION_TRIGGER_TYPE_KEY }; + this.keyRequired = new int[] { 1, 0, 0 }; } @Override @@ -94,8 +94,16 @@ public NounMetadata execute() { Map inputsMap = this.getMap(AutomationConstants.AUTOMATION_INPUTS_KEY); AutomationExecutionUtils.applyPlaygroundInputs(ordered, inputsMap); + String triggerType = this.keyValue.get(AutomationConstants.AUTOMATION_TRIGGER_TYPE_KEY); + if (triggerType == null || triggerType.isBlank()) { + triggerType = AutomationConstants.TRIGGER_MANUAL; + } else if (!AutomationConstants.TRIGGER_MANUAL.equals(triggerType) + && !AutomationConstants.TRIGGER_PLAYGROUND.equals(triggerType)) { + classLogger.warn("Unknown triggerType '{}' for run {}, defaulting to MANUAL", triggerType, runId); + triggerType = AutomationConstants.TRIGGER_MANUAL; + } AutomationDatabaseUtility.insertRun(runId, projectId, AutomationConstants.DEFAULT_AUTOMATION_ID, - AutomationConstants.TRIGGER_MANUAL, ordered.size(), userId); + triggerType, ordered.size(), userId); AutomationDatabaseUtility.insertAllNodeOutputs(runId, ordered); classLogger.info("Automation run {} starting for project {}", runId, projectId); @@ -223,6 +231,7 @@ public String getReactorDescription() { protected String getDescriptionForKey(String key) { if (ReactorKeysEnum.PROJECT.getKey().equals(key)) return "The project (app) ID or alias to run the automation for."; if (AutomationConstants.AUTOMATION_INPUTS_KEY.equals(key)) return "Optional map of playground-supplied values to inject into automation node fields before running. Keys are parameter names from the automation's MCP tool schema."; + if (AutomationConstants.AUTOMATION_TRIGGER_TYPE_KEY.equals(key)) return "Caller-supplied trigger source. Defaults to MANUAL when omitted; MCP/playground callers pass PLAYGROUND."; return super.getDescriptionForKey(key); } } From dc2f33baf932014feeaf86a7f5564d2dcc86d158 Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Thu, 6 Aug 2026 08:02:00 -0400 Subject: [PATCH 16/25] feat: improve automation user-friendlyness --- .../automation/AutomationConstants.java | 7 +- .../automation/AutomationDatabaseUtility.java | 43 +- .../reactor/automation/AutomationMcpSync.java | 23 +- .../automation/GenerateAutomationReactor.java | 593 ++++++++++++++++++ .../automation/GenerateSQLReactor.java | 244 +++++++ .../GetReactorSignatureReactor.java | 227 +++++++ .../automation/TriggerAutomationReactor.java | 29 +- .../scheduler/SchedulerOwlCreator.java | 3 +- 8 files changed, 1148 insertions(+), 21 deletions(-) create mode 100644 src/prerna/reactor/automation/GenerateAutomationReactor.java create mode 100644 src/prerna/reactor/automation/GenerateSQLReactor.java create mode 100644 src/prerna/reactor/automation/GetReactorSignatureReactor.java diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index 21abb3adc68..cc85392d151 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -60,6 +60,7 @@ private AutomationConstants() {} public static final String COMPLETED_NODES = "COMPLETED_NODES"; public static final String CREATED_BY = "CREATED_BY"; public static final String CANCEL_REQUESTED = "CANCEL_REQUESTED"; + public static final String RESULT_SUMMARY_COL = "RESULT_SUMMARY"; // -- AUTOMATION_ACTIVE_RUN columns --------------------------------------------- @@ -163,12 +164,8 @@ private AutomationConstants() {} public static final String DOC_NODES = "nodes"; public static final String DOC_EDGES = "edges"; public static final int DOC_CURRENT_VERSION = 1; - /** - * Optional {@code ${var}}/{@code ${config.KEY}} template resolved against the final run - * scope once all nodes complete, producing a workflow-specific human-readable summary - * (e.g. "Indexed 20 files") instead of a raw JSON blob for MCP/agent consumers. - */ public static final String DOC_RESULT_MESSAGE_TEMPLATE = "resultMessageTemplate"; + public static final String DOC_DESCRIPTION = "description"; // -- Node/edge field names -------------------------------------------------------- diff --git a/src/prerna/reactor/automation/AutomationDatabaseUtility.java b/src/prerna/reactor/automation/AutomationDatabaseUtility.java index ec6e82e5fbe..37df201ff7a 100644 --- a/src/prerna/reactor/automation/AutomationDatabaseUtility.java +++ b/src/prerna/reactor/automation/AutomationDatabaseUtility.java @@ -30,6 +30,7 @@ import static prerna.reactor.automation.AutomationConstants.AUTOMATION_ID; import static prerna.reactor.automation.AutomationConstants.BIGINT; import static prerna.reactor.automation.AutomationConstants.CANCEL_REQUESTED; +import static prerna.reactor.automation.AutomationConstants.RESULT_SUMMARY_COL; import static prerna.reactor.automation.AutomationConstants.CLAIMED_AT; import static prerna.reactor.automation.AutomationConstants.COMPLETED_AT; import static prerna.reactor.automation.AutomationConstants.COMPLETED_NODES; @@ -139,6 +140,9 @@ private AutomationDatabaseUtility() { UPDATE AUTOMATION_RUNS SET STATUS = ?, COMPLETED_AT = ?, \ FAILED_NODE_ID = ?, ERROR_MESSAGE = ? WHERE RUN_ID = ?"""; + private static final String UPDATE_RUN_SUMMARY = + "UPDATE AUTOMATION_RUNS SET RESULT_SUMMARY = ? WHERE RUN_ID = ?"; + private static final String UPDATE_HEARTBEAT = "UPDATE AUTOMATION_RUNS SET LAST_HEARTBEAT = ?, COMPLETED_NODES = ? WHERE RUN_ID = ?"; @@ -546,6 +550,35 @@ public static boolean updateRunStatus(String runId, String status, } } + /** + * Persists the human-readable outcome summary for a completed run. + * Called after the run finishes, separately from {@link #updateRunStatus} because the + * summary is built by the caller ({@code TriggerAutomationReactor}) after the engine returns. + */ + public static boolean updateRunSummary(String runId, String resultSummary) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(UPDATE_RUN_SUMMARY)) { + setNullableString(ps, 1, resultSummary); + ps.setString(2, runId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to update run summary for '{}': {}", runId, e.getMessage(), e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + /** * Updates the heartbeat timestamp and completed node count for a running automation. */ @@ -626,6 +659,7 @@ public static List> getRunsForProject(String projectId, int qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + TOTAL_NODES, TOTAL_NODES)); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + COMPLETED_NODES, COMPLETED_NODES)); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + CREATED_BY, CREATED_BY)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + RESULT_SUMMARY_COL, RESULT_SUMMARY_COL)); qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( TABLE_RUNS + "__" + PROJECT_ID, "==", projectId, PixelDataType.CONST_STRING)); @@ -657,6 +691,7 @@ public static Map getRunDetail(String runId) { qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + TOTAL_NODES, TOTAL_NODES)); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + COMPLETED_NODES, COMPLETED_NODES)); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + CREATED_BY, CREATED_BY)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + RESULT_SUMMARY_COL, RESULT_SUMMARY_COL)); qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( TABLE_RUNS + "__" + RUN_ID, "==", runId, PixelDataType.CONST_STRING)); @@ -854,13 +889,13 @@ private static void createAutomationRunsTable(Connection conn, AbstractSqlQueryU String[] colNames = { RUN_ID, PROJECT_ID, AUTOMATION_ID, STATUS, TRIGGER_TYPE, STARTED_AT, COMPLETED_AT, FAILED_NODE_ID, ERROR_MESSAGE, LAST_HEARTBEAT, - TOTAL_NODES, COMPLETED_NODES, CREATED_BY, CANCEL_REQUESTED }; + TOTAL_NODES, COMPLETED_NODES, CREATED_BY, CANCEL_REQUESTED, RESULT_SUMMARY_COL }; String[] types = { VARCHAR_255, VARCHAR_255, VARCHAR_255, VARCHAR_50, VARCHAR_50, dateTimeType, dateTimeType, VARCHAR_255, clobType, dateTimeType, - INTEGER, INTEGER, VARCHAR_255, queryUtil.getBooleanDataTypeName() }; + INTEGER, INTEGER, VARCHAR_255, queryUtil.getBooleanDataTypeName(), VARCHAR_2000 }; String[] constraints = { NOT_NULL, NOT_NULL, null, NOT_NULL, NOT_NULL, NOT_NULL, null, null, null, null, - null, null, null, null }; + null, null, null, null, null }; String sql; if (allowIfExists) { @@ -875,6 +910,8 @@ private static void createAutomationRunsTable(Connection conn, AbstractSqlQueryU // Migrate installs that predate cluster-safe cancel addColumnIfNotExists(conn, queryUtil, tableName, CANCEL_REQUESTED, queryUtil.getBooleanDataTypeName()); + // Migrate installs that predate result summary + addColumnIfNotExists(conn, queryUtil, tableName, RESULT_SUMMARY_COL, VARCHAR_2000); // Primary key addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, PK_AUTOMATION_RUNS, diff --git a/src/prerna/reactor/automation/AutomationMcpSync.java b/src/prerna/reactor/automation/AutomationMcpSync.java index fe05505afe8..b09c1ea5994 100644 --- a/src/prerna/reactor/automation/AutomationMcpSync.java +++ b/src/prerna/reactor/automation/AutomationMcpSync.java @@ -127,8 +127,27 @@ private static JSONObject buildTriggerAutomationTool(String projectId, String au JSONObject tool = new JSONObject(); tool.put("name", "TriggerAutomation"); tool.put("title", "Trigger Automation"); - String description = "Manually triggers the automation configured for this project/app and returns a " - + "per-workflow summary once complete (e.g. \"Indexed 20 files\")."; + + String docDescription = null; + try { + if (automationJson != null && !automationJson.isBlank()) { + JSONObject doc = new JSONObject(automationJson); + String raw = doc.optString(AutomationConstants.DOC_DESCRIPTION, "").trim(); + if (!raw.isEmpty()) { + docDescription = raw; + } + } + } catch (Exception e) { + classLogger.warn("Failed to read description from automation JSON for project {}", projectId, e); + } + + String description; + if (docDescription != null) { + description = docDescription + " Triggers the automation and returns a per-workflow summary once complete."; + } else { + description = "Manually triggers the automation configured for this project/app and returns a " + + "per-workflow summary once complete (e.g. \"Indexed 20 files\")."; + } if (hasDbNodes) { description += " This automation has database nodes that accept SQL queries — call GetAutomationSchema first" + " to discover the exact table and column names before writing SQL."; diff --git a/src/prerna/reactor/automation/GenerateAutomationReactor.java b/src/prerna/reactor/automation/GenerateAutomationReactor.java new file mode 100644 index 00000000000..ff619c35e90 --- /dev/null +++ b/src/prerna/reactor/automation/GenerateAutomationReactor.java @@ -0,0 +1,593 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.engine.api.IDatabaseEngine; +import prerna.engine.api.IModelEngine; +import prerna.engine.api.IRDBMSEngine; +import prerna.project.api.IProject; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Uses an LLM to scaffold a starter {@link AutomationDocument} from a plain-English description. + * + *

Pixel: {@code GenerateAutomation(project=["appId"], description=["what it should do"], engine=["modelEngineId"])} + * + *

Generation is two-pass: pass 1 builds the workflow structure and picks engines; pass 2 fetches + * the real DB schema for any chosen database engines and rewrites SQL expressions and model commands + * with accurate table/column names. Pass 2 is skipped when no database-engine nodes exist. + * + *

Returns the generated JSON as a string (same shape as {@code GetAutomation}). The caller is + * expected to display it for review and save it via {@code SaveAutomation} — this reactor does NOT + * persist anything. + */ +public final class GenerateAutomationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GenerateAutomationReactor.class); + + private static final int DESCRIPTION_MAX_CHARS = 1000; + private static final String PENDING_SQL = "PENDING_SQL_GENERATION"; + + /** Engine types used to fetch context for the prompt. */ + private static final List SUPPORTED_ENGINE_TYPES = Arrays.asList( + "DATABASE", "MODEL", "VECTOR", "STORAGE", "FUNCTION"); + + /** + * Pass 1 system prompt: structural generation only. + * The LLM picks node types, engines, labels, and output vars. + * DB expressions are set to PENDING_SQL_GENERATION for pass 2 to fill in. + */ + private static final String SYSTEM_PROMPT_STRUCTURAL = """ +You are a workflow builder assistant. Generate an automation graph JSON document from the user's plain-English description. + +## Available node types +Each node has these fields: id (string), type (string), label (string), position ({x:0,y:0}), outputVar (string), config (object). + +Node types and their config shapes: +- trigger: config={"mode":"manual"} — always the first node, outputVar="trigger_out" +- database-engine: config={"engineId":"","operation":"query","expression":"PENDING_SQL_GENERATION","limit":50,"commit":false} — SQL queries, outputVar="db_out" +- model-engine: config={"engineId":"","operation":"llm","command":"","context":"","paramValues":"","values":"","image":"","prompt":"","entities":""} — LLM calls, outputVar="model_out" +- vector-engine: config={"engineId":"","operation":"search","command":"","limit":5,"filters":"","metaFilters":"","filePath":"","source":"","space":"","filePaths":"","paramValues":"","fileNames":""} — semantic search, outputVar="vector_out" +- storage-engine: config={"engineId":"","operation":"list","storagePath":"/","filePath":"","metadata":""} — file storage, outputVar="storage_out" +- function-engine: config={"engineId":"","operation":"execute","params":""} — custom functions, outputVar="fn_out" +- app: config={"pixel":"PENDING_PIXEL_EXPRESSION","appId":""} — run a custom reactor or arbitrary Pixel, outputVar="pixel_out" +- wait: config={"seconds":"5"} — pause between steps, outputVar="wait_out" + +## Variable substitution +Reference upstream node outputs in config fields using ${outputVar}. + +## Rules +1. Always start with a trigger node (id="trigger-1"). +2. Use only node types from the list above. +3. Set engineId from the available engines listed below. Leave empty string "" if no suitable engine exists. +4. Keep outputVar names unique and descriptive of what the node produces. +5. Make node labels action-oriented and specific to what the node does. +6. Build a realistic, useful graph — don't add unnecessary nodes. +7. For database-engine nodes: set expression to exactly "PENDING_SQL_GENERATION" — a second pass will fill in real SQL using the actual schema. +8. For app nodes: set pixel to exactly "PENDING_PIXEL_EXPRESSION" — a second pass will fill in the reactor call using the project's available reactors. +9. For model-engine nodes: write a concrete instruction in "command" appropriate to the user's request, referencing upstream outputVars where relevant. Exact column names are not yet known; describe the intent clearly. +10. Respond with ONLY valid JSON. No markdown, no code fences, no explanation. + +## Response format +{"version":1,"description":"","graph":{"nodes":[...],"edges":[]}} +"""; + + /** + * Edit/iterate mode system prompt: same structure rules as pass 1, but instructs the LLM to + * treat the input as an existing document to modify rather than generate from scratch. + */ + private static final String SYSTEM_PROMPT_EDIT = """ +You are a workflow builder assistant. You are given an existing automation graph JSON document. Modify it based on the user's request. Preserve all steps and structure that are not directly affected by the request. You may add, remove, or change nodes as needed. Always keep a trigger node as the first node. + +## Available node types +Each node has these fields: id (string), type (string), label (string), position ({x:0,y:0}), outputVar (string), config (object). + +Node types and their config shapes: +- trigger: config={"mode":"manual"} — always the first node, outputVar="trigger_out" +- database-engine: config={"engineId":"","operation":"query","expression":"PENDING_SQL_GENERATION","limit":50,"commit":false} — SQL queries, outputVar="db_out" +- model-engine: config={"engineId":"","operation":"llm","command":"","context":"","paramValues":"","values":"","image":"","prompt":"","entities":""} — LLM calls, outputVar="model_out" +- vector-engine: config={"engineId":"","operation":"search","command":"","limit":5,"filters":"","metaFilters":"","filePath":"","source":"","space":"","filePaths":"","paramValues":"","fileNames":""} — semantic search, outputVar="vector_out" +- storage-engine: config={"engineId":"","operation":"list","storagePath":"/","filePath":"","metadata":""} — file storage, outputVar="storage_out" +- function-engine: config={"engineId":"","operation":"execute","params":""} — custom functions, outputVar="fn_out" +- app: config={"pixel":"PENDING_PIXEL_EXPRESSION","appId":""} — run a custom reactor or arbitrary Pixel, outputVar="pixel_out" +- wait: config={"seconds":"5"} — pause between steps, outputVar="wait_out" + +## Variable substitution +Reference upstream node outputs in config fields using ${outputVar}. + +## Rules +1. Always start with a trigger node (id="trigger-1"). +2. Use only node types from the list above. +3. Set engineId from the available engines listed below. Leave empty string "" if no suitable engine exists. +4. Keep outputVar names unique and descriptive of what the node produces. +5. Make node labels action-oriented and specific to what the node does. +6. Build a realistic, useful graph — don't add unnecessary nodes. +7. For database-engine nodes: set expression to exactly "PENDING_SQL_GENERATION" — a second pass will fill in real SQL using the actual schema. +8. For app nodes: set pixel to exactly "PENDING_PIXEL_EXPRESSION" — a second pass will fill in the reactor call using the project's available reactors. +9. For model-engine nodes: write a concrete instruction in "command" appropriate to the user's request, referencing upstream outputVars where relevant. Exact column names are not yet known; describe the intent clearly. +10. Respond with ONLY valid JSON. No markdown, no code fences, no explanation. + +## Response format +{"version":1,"description":"","graph":{"nodes":[...],"edges":[]}} +"""; + + /** + * Pass 2 system prompt: schema-aware refinement. + * Rewrites PENDING_SQL_GENERATION, PENDING_PIXEL_EXPRESSION, and model commands using actual schema and reactor list. + */ + private static final String SYSTEM_PROMPT_REFINE = """ +You are given a workflow automation document along with context gathered after pass 1: the actual database schemas for chosen database engines, and the list of custom reactors available in this project. + +Your task: return a corrected copy of the document with these updates: +1. database-engine nodes: replace the "expression" value "PENDING_SQL_GENERATION" with a real SQL SELECT query using the actual table and column names from the schema. + - Write SQL that matches what the user asked for — use appropriate joins, filters, and columns based on the user's intent + - Do not add conditions or filters that aren't implied by the user's request + - Use column types to write type-safe SQL — don't compare VARCHAR columns to integers or assume a column's semantics from its name alone + - Use a reasonable LIMIT to avoid returning unbounded result sets + - If no relevant table exists in the schema, set expression to "-- [replace with your SQL query]" and update the label to "Review: update this query" +2. model-engine nodes: if the "command" references an upstream database node's output, update it to mention the specific column names the SQL query will actually return. +3. app nodes: replace the "pixel" value "PENDING_PIXEL_EXPRESSION" with a call to the most appropriate reactor from the available reactors list. + - Use the format: ReactorName(param="${varName}") referencing upstream outputVars where relevant + - If no reactor in the list clearly matches the intent, set pixel to "-- [describe the Pixel expression to write here]" + +Keep ALL other fields (ids, types, labels, outputVars, engineIds, edges, etc.) EXACTLY as they are. + +Respond with ONLY the complete updated JSON document. No markdown, no code fences, no explanation. +"""; + + /** Key for an optional base64-encoded current doc to modify (edit/iterate mode). */ + private static final String CURRENT_DOC_KEY = "currentDoc"; + + public GenerateAutomationReactor() { + this.keysToGet = new String[] { + ReactorKeysEnum.PROJECT.getKey(), + AutomationConstants.DOC_DESCRIPTION, + ReactorKeysEnum.ENGINE.getKey(), + CURRENT_DOC_KEY + }; + this.keyRequired = new int[] { 1, 1, 0, 0 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } + + String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); + String description = this.keyValue.get(AutomationConstants.DOC_DESCRIPTION); + String engineId = this.keyValue.get(ReactorKeysEnum.ENGINE.getKey()); + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); + // View access is intentional — this reactor is read-only (no persistence). + // The caller saves the result via SaveAutomation, which enforces edit access. + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access."); + } + + if (description == null || description.trim().isEmpty()) { + throw new IllegalArgumentException("A description of what the automation should do is required."); + } + // Decode base64-encoded description sent by the FE to prevent Pixel injection + try { + description = new String( + java.util.Base64.getDecoder().decode(description.trim()), + java.nio.charset.StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + // Not base64-encoded (e.g. direct API call) — use the value as-is + } + if (description.length() > DESCRIPTION_MAX_CHARS) { + description = description.substring(0, DESCRIPTION_MAX_CHARS); + } + + // Resolve model engine — use provided ID or fall back to first available MODEL engine + if (engineId == null || engineId.trim().isEmpty()) { + engineId = findFirstModelEngine(user); + } + if (engineId == null || engineId.trim().isEmpty()) { + throw new IllegalArgumentException( + "No AI model engine is available. Add a model engine connection to generate an automation."); + } + if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { + throw new IllegalArgumentException( + "Model engine " + engineId + " does not exist or user does not have access."); + } + + IModelEngine modelEngine = Utility.getModel(engineId); + if (modelEngine == null) { + throw new IllegalArgumentException( + "Model engine " + engineId + " could not be loaded. It may no longer exist."); + } + + // Decode optional current doc (edit/iterate mode) + String currentDocRaw = this.keyValue.get(CURRENT_DOC_KEY); + String currentDoc = null; + if (currentDocRaw != null && !currentDocRaw.trim().isEmpty()) { + try { + currentDoc = new String( + java.util.Base64.getDecoder().decode(currentDocRaw.trim()), + java.nio.charset.StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + // Not base64-encoded (e.g. direct API call) — use the value as-is + currentDoc = currentDocRaw; + } + if (currentDoc != null && currentDoc.length() > 50_000) { + currentDoc = currentDoc.substring(0, 50_000); + } + } + + Map paramMap = new HashMap<>(); + paramMap.put("use_history", false); + + // -- Pass 1: structural generation (or edit if current doc provided) ------------- + String enginesSection = buildAvailableEnginesSection(user); + String pass1Message; + String systemPrompt1; + if (currentDoc != null) { + classLogger.info("GenerateAutomation edit mode: project={}, docLength={}", projectId, currentDoc.length()); + systemPrompt1 = SYSTEM_PROMPT_EDIT; + pass1Message = enginesSection + + "\n## Existing automation\n```json\n" + currentDoc + "\n```" + + "\n## User's modification request\n" + description.trim(); + } else { + systemPrompt1 = SYSTEM_PROMPT_STRUCTURAL; + pass1Message = enginesSection + "\n## User's request\n" + description.trim(); + } + String raw = callLlm(modelEngine, systemPrompt1, pass1Message, paramMap, projectId); + raw = stripCodeFences(raw); + validateGeneratedDoc(raw); + + // -- Pass 2: schema-aware refinement (when DB nodes or app nodes are present) ---- + List dbEngineIds = extractDatabaseEngineIds(raw); + boolean hasAppNodes = extractHasAppNodes(raw); + if (!dbEngineIds.isEmpty() || hasAppNodes) { + classLogger.info("GenerateAutomation pass 2: db engines={}, appNodes={}, project={}", + dbEngineIds.size(), hasAppNodes, projectId); + StringBuilder pass2Message = new StringBuilder("## Workflow document from pass 1\n").append(raw).append("\n"); + if (!dbEngineIds.isEmpty()) { + pass2Message.append("\n").append(buildSchemaForEngineIds(dbEngineIds)); + } + if (hasAppNodes) { + pass2Message.append("\n").append(buildReactorListSection(projectId)); + } + try { + String refined = callLlm(modelEngine, SYSTEM_PROMPT_REFINE, pass2Message.toString(), paramMap, projectId); + refined = stripCodeFences(refined); + validateGeneratedDoc(refined); + raw = refined; + } catch (Exception e) { + classLogger.warn("GenerateAutomation pass 2 failed — returning pass 1 result. Reason: {}", e.getMessage()); + // Fall through: return pass 1 result unchanged + } + } + + return new NounMetadata(raw, PixelDataType.CONST_STRING, PixelOperationType.OPERATION); + } + + // -- Private helpers ------------------------------------------------------------- + + /** Calls the model and returns the response text, throwing on failure. */ + private String callLlm(IModelEngine modelEngine, String systemPrompt, String userMessage, + Map paramMap, String projectId) { + Map response; + try { + response = modelEngine.ask(systemPrompt + "\n\n" + userMessage, null, this.insight, paramMap).toMap(); + } catch (Exception e) { + classLogger.error("LLM call failed for GenerateAutomation on project {}", projectId, e); + throw new RuntimeException("AI generation failed: " + e.getMessage(), e); + } + String text = extractResponseText(response); + if (text == null || text.isBlank()) { + throw new IllegalStateException( + "The AI model did not return a response. Try again or start with a blank automation."); + } + return text; + } + + /** + * Parses a generated document and returns the engineIds of all database-engine nodes + * that have a non-blank engineId. Used to decide whether pass 2 is needed. + */ + private static List extractDatabaseEngineIds(String docJson) { + List ids = new ArrayList<>(); + try { + JSONObject doc = new JSONObject(docJson); + JSONObject graph = doc.optJSONObject("graph"); + if (graph == null) return ids; + JSONArray nodes = graph.optJSONArray("nodes"); + if (nodes == null) return ids; + for (int i = 0; i < nodes.length(); i++) { + JSONObject node = nodes.optJSONObject(i); + if (node == null) continue; + if ("database-engine".equals(node.optString("type"))) { + JSONObject config = node.optJSONObject("config"); + if (config != null) { + String dbId = config.optString("engineId", "").trim(); + if (!dbId.isEmpty() && !ids.contains(dbId)) { + ids.add(dbId); + } + } + } + } + } catch (Exception e) { + classLogger.warn("Failed to extract database engine IDs from generated doc", e); + } + return ids; + } + + /** + * Returns true if the document contains any app-type nodes whose pixel is still PENDING_PIXEL_EXPRESSION. + * Used to decide whether pass 2 needs to include the reactor list. + */ + private static boolean extractHasAppNodes(String docJson) { + try { + JSONObject doc = new JSONObject(docJson); + JSONObject graph = doc.optJSONObject("graph"); + if (graph == null) return false; + JSONArray nodes = graph.optJSONArray("nodes"); + if (nodes == null) return false; + for (int i = 0; i < nodes.length(); i++) { + JSONObject node = nodes.optJSONObject(i); + if (node == null) continue; + if ("app".equals(node.optString("type"))) { + JSONObject config = node.optJSONObject("config"); + if (config != null && "PENDING_PIXEL_EXPRESSION".equals(config.optString("pixel"))) { + return true; + } + } + } + } catch (Exception e) { + classLogger.warn("Failed to check for app nodes in generated doc", e); + } + return false; + } + + /** + * Returns the list of custom reactors available in the given project, formatted as a + * bulleted list for the pass 2 prompt. Uses the same source as the FE reactor browser. + */ + private static String buildReactorListSection(String projectId) { + StringBuilder sb = new StringBuilder("## Available custom reactors in this project\n"); + try { + IProject project = Utility.getProject(projectId); + if (project == null) { + sb.append("(project not found — leave app node pixel as-is)\n"); + return sb.toString(); + } + java.util.TreeSet reactors = project.getAvailableReactors(); + if (reactors == null || reactors.isEmpty()) { + sb.append("(none — leave app node pixel as a comment placeholder)\n"); + return sb.toString(); + } + for (String name : reactors) { + sb.append("- ").append(name).append("\n"); + } + } catch (Exception e) { + classLogger.warn("Failed to build reactor list for project {}", projectId, e); + sb.append("(reactor list unavailable — leave app node pixel as a comment placeholder)\n"); + } + return sb.toString(); + } + + /** + * Fetches the table/column schema (with data types) for the given engine IDs. + * Uses the same metamodel API as {@code TextToSQLReactor}. Data types prevent the LLM + * from generating type-mismatched WHERE clauses (e.g. comparing VARCHAR to an integer). + */ + private static String buildSchemaForEngineIds(List engineIds) { + StringBuilder sb = new StringBuilder("## Database schema\n"); + for (String engineId : engineIds) { + try { + IDatabaseEngine dbEngine = Utility.getDatabase(engineId); + if (!(dbEngine instanceof IRDBMSEngine rdbms)) { + sb.append("Database id=\"").append(engineId).append("\": (non-relational — no table schema)\n"); + continue; + } + sb.append("Database id=\"").append(engineId).append("\":\n"); + List tables = rdbms.getPixelConcepts(); + if (tables == null || tables.isEmpty()) { + sb.append(" (no tables found)\n"); + continue; + } + for (String table : tables) { + List columns = rdbms.getPixelSelectors(table); + sb.append(" - ").append(table).append(" ("); + if (columns != null && !columns.isEmpty()) { + StringBuilder cols = new StringBuilder(); + for (String col : columns) { + int sep = col.lastIndexOf("__"); + String colName = sep >= 0 ? col.substring(sep + 2) : col; + String dataType = null; + try { + String physicalUri = rdbms.getPhysicalUriFromPixelSelector(col); + if (physicalUri != null) { + dataType = rdbms.getDataTypes(physicalUri); + } + } catch (Exception ignored) { + // Data type fetch is best-effort; column name alone is still useful + } + if (cols.length() > 0) cols.append(", "); + cols.append(colName); + if (dataType != null && !dataType.isBlank()) { + cols.append(": ").append(dataType.trim()); + } + } + sb.append(cols); + } + sb.append(")\n"); + } + } catch (Exception e) { + classLogger.warn("Failed to fetch schema for engine {}", engineId, e); + sb.append("Database id=\"").append(engineId).append("\": (schema unavailable)\n"); + } + } + return sb.toString(); + } + + /** + * Returns the ID of the first MODEL-type engine the user has access to, or null if none. + */ + private static String findFirstModelEngine(User user) { + try { + List> engines = SecurityEngineUtils.getUserEngineList( + user, List.of("MODEL"), null, false, null, null, null, "1", "0", null); + if (engines != null && !engines.isEmpty()) { + Object id = engines.get(0).get("database_id"); + return id != null ? String.valueOf(id) : null; + } + } catch (Exception e) { + classLogger.warn("Failed to auto-discover model engine", e); + } + return null; + } + + /** + * Builds the "## Available engines" section appended to the pass 1 user message. + * Lists each engine the user has access to so the LLM can populate engineId fields. + */ + private static String buildAvailableEnginesSection(User user) { + StringBuilder sb = new StringBuilder("## Available engines\n"); + try { + List> engines = SecurityEngineUtils.getUserEngineList( + user, SUPPORTED_ENGINE_TYPES, null, false, null, null, null, "50", "0", null); + if (engines == null || engines.isEmpty()) { + sb.append("None available — leave engineId as empty string.\n"); + return sb.toString(); + } + for (Map engine : engines) { + String id = engine.getOrDefault("database_id", "").toString(); + String name = engine.getOrDefault("database_name", "").toString(); + String type = engine.getOrDefault("engine_type", "").toString().toUpperCase(); + sb.append("- type=").append(type) + .append(" id=\"").append(id) + .append("\" name=\"").append(name).append("\"\n"); + } + } catch (Exception e) { + classLogger.warn("Failed to build engine list for generation prompt", e); + sb.append("(engine list unavailable — leave engineId as empty string)\n"); + } + return sb.toString(); + } + + /** + * Extracts the text content from the model's response map. + * Handles both {@code response} string and {@code output}/{@code content} keys. + */ + private static String extractResponseText(Map response) { + if (response == null) return null; + Object resp = response.get("response"); + if (resp instanceof String s && !s.isBlank()) return s; + Object output = response.get("output"); + if (output instanceof String s && !s.isBlank()) return s; + Object content = response.get("content"); + if (content instanceof String s && !s.isBlank()) return s; + return null; + } + + /** Strips leading/trailing markdown code fences (```json ... ``` or ``` ... ```). */ + private static String stripCodeFences(String raw) { + String trimmed = raw.strip(); + if (trimmed.startsWith("```")) { + int firstNewline = trimmed.indexOf('\n'); + if (firstNewline != -1) { + trimmed = trimmed.substring(firstNewline + 1); + } + if (trimmed.endsWith("```")) { + trimmed = trimmed.substring(0, trimmed.lastIndexOf("```")).strip(); + } + } + return trimmed; + } + + /** + * Throws if the generated doc is not parseable JSON or missing required structure. + * Does not enforce strict schema — just enough to avoid crashing the FE. + */ + private static void validateGeneratedDoc(String raw) { + try { + JSONObject doc = new JSONObject(raw); + if (!doc.has("graph")) { + throw new IllegalStateException("Generated document is missing the 'graph' field."); + } + JSONObject graph = doc.getJSONObject("graph"); + if (!graph.has("nodes")) { + throw new IllegalStateException("Generated graph is missing the 'nodes' array."); + } + JSONArray nodes = graph.getJSONArray("nodes"); + if (nodes.length() == 0) { + throw new IllegalStateException("Generated graph has no nodes."); + } + } catch (org.json.JSONException e) { + classLogger.warn("Generated automation doc is not valid JSON: {}", raw, e); + throw new IllegalStateException( + "The AI model returned an invalid response. Please try again with a different description.", e); + } + } + + @Override + public String getReactorDescription() { + return "Uses an AI model to scaffold a starter automation graph from a plain-English description. " + + "Returns the generated document JSON — the caller must save it via SaveAutomation."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (AutomationConstants.DOC_DESCRIPTION.equals(key)) { + return "Plain-English description of what the automation should do (max 1000 characters)."; + } else if (ReactorKeysEnum.PROJECT.getKey().equals(key)) { + return "Project ID that will own this automation."; + } else if (ReactorKeysEnum.ENGINE.getKey().equals(key)) { + return "Optional model engine ID to use for generation. Defaults to the first available MODEL engine."; + } else if (CURRENT_DOC_KEY.equals(key)) { + return "Optional base64-encoded JSON of an existing automation document. When provided, the LLM modifies the existing document rather than generating from scratch."; + } + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/reactor/automation/GenerateSQLReactor.java b/src/prerna/reactor/automation/GenerateSQLReactor.java new file mode 100644 index 00000000000..09f5059852a --- /dev/null +++ b/src/prerna/reactor/automation/GenerateSQLReactor.java @@ -0,0 +1,244 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.engine.api.IDatabaseEngine; +import prerna.engine.api.IModelEngine; +import prerna.engine.api.IRDBMSEngine; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Uses an LLM to generate a SQL SELECT query for a given database engine from a plain-English description. + * + *

Pixel: {@code GenerateSQL(database=["engineId"], description=["base64EncodedDescription"])} + * + *

Fetches the engine's schema, finds the first available MODEL engine, and returns a SQL string. + * The caller is expected to review and edit the result before running it. + */ +public final class GenerateSQLReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GenerateSQLReactor.class); + + private static final String SYSTEM_PROMPT = """ +You are a SQL expert. Given a database schema and a plain-English description, write a single SQL SELECT query that retrieves the requested data. + +Rules: +- Use only the tables and columns that exist in the schema +- Use column types to write type-safe SQL — don't compare VARCHAR columns to integers +- Do not add conditions or filters not implied by the description +- Include a LIMIT clause appropriate to the request (e.g. LIMIT 100 for "get all X" queries) +- Return ONLY the SQL statement — no markdown, no explanation, no code fences + +## Database schema +"""; + + public GenerateSQLReactor() { + this.keysToGet = new String[] { + ReactorKeysEnum.DATABASE.getKey(), + AutomationConstants.DOC_DESCRIPTION, + ReactorKeysEnum.ENGINE.getKey() + }; + this.keyRequired = new int[] { 1, 1, 0 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } + + String databaseId = this.keyValue.get(ReactorKeysEnum.DATABASE.getKey()); + String description = this.keyValue.get(AutomationConstants.DOC_DESCRIPTION); + String engineId = this.keyValue.get(ReactorKeysEnum.ENGINE.getKey()); + + if (databaseId == null || databaseId.trim().isEmpty()) { + throw new IllegalArgumentException("A database engine ID is required."); + } + if (!SecurityEngineUtils.userCanViewEngine(user, databaseId)) { + throw new IllegalArgumentException("Database engine does not exist or user does not have access."); + } + + if (description == null || description.trim().isEmpty()) { + throw new IllegalArgumentException("A description of the data to retrieve is required."); + } + // Decode base64-encoded description from FE + try { + description = new String( + java.util.Base64.getDecoder().decode(description.trim()), + java.nio.charset.StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + // Not base64-encoded — use as-is + } + + // Resolve model engine + if (engineId == null || engineId.trim().isEmpty()) { + engineId = findFirstModelEngine(user); + } + if (engineId == null || engineId.trim().isEmpty()) { + throw new IllegalArgumentException( + "No AI model engine is available. Add a model engine connection to use SQL generation."); + } + if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { + throw new IllegalArgumentException("Model engine does not exist or user does not have access."); + } + + IModelEngine modelEngine = Utility.getModel(engineId); + if (modelEngine == null) { + throw new IllegalArgumentException("Model engine could not be loaded."); + } + + // Build schema section + String schema = buildSchemaForEngine(databaseId); + + // Call LLM + String prompt = SYSTEM_PROMPT + schema + "\n## Request\n" + description.trim(); + Map paramMap = new HashMap<>(); + paramMap.put("use_history", false); + + Map response; + try { + response = modelEngine.ask(prompt, null, this.insight, paramMap).toMap(); + } catch (Exception e) { + classLogger.error("LLM call failed for GenerateSQL on database {}", databaseId, e); + throw new RuntimeException("SQL generation failed: " + e.getMessage(), e); + } + + String sql = extractResponseText(response); + if (sql == null || sql.isBlank()) { + throw new IllegalStateException("The AI model did not return a response. Try again."); + } + sql = stripCodeFences(sql).trim(); + + return new NounMetadata(sql, PixelDataType.CONST_STRING, PixelOperationType.OPERATION); + } + + private static String buildSchemaForEngine(String engineId) { + StringBuilder sb = new StringBuilder(); + try { + IDatabaseEngine dbEngine = Utility.getDatabase(engineId); + if (!(dbEngine instanceof IRDBMSEngine rdbms)) { + return "(non-relational engine — no table schema available)\n"; + } + List tables = rdbms.getPixelConcepts(); + if (tables == null || tables.isEmpty()) { + return "(no tables found)\n"; + } + for (String table : tables) { + List columns = rdbms.getPixelSelectors(table); + sb.append(table).append(" ("); + if (columns != null && !columns.isEmpty()) { + StringBuilder cols = new StringBuilder(); + for (String col : columns) { + String colName = col.contains("__") ? col.split("__")[1] : col; + String dataType = null; + try { + String physUri = rdbms.getPhysicalUriFromPixelSelector(col); + if (physUri != null) dataType = rdbms.getDataTypes(physUri); + } catch (Exception ignored) { } + if (cols.length() > 0) cols.append(", "); + cols.append(colName); + if (dataType != null && !dataType.isBlank()) { + cols.append(": ").append(dataType.trim()); + } + } + sb.append(cols); + } + sb.append(")\n"); + } + } catch (Exception e) { + classLogger.warn("Failed to fetch schema for engine {}", engineId, e); + return "(schema unavailable)\n"; + } + return sb.toString(); + } + + private static String findFirstModelEngine(User user) { + try { + List> engines = SecurityEngineUtils.getUserEngineList( + user, List.of("MODEL"), null, false, null, null, null, "1", "0", null); + if (engines != null && !engines.isEmpty()) { + Object id = engines.get(0).get("database_id"); + return id != null ? String.valueOf(id) : null; + } + } catch (Exception e) { + classLogger.warn("Failed to auto-discover model engine for GenerateSQL", e); + } + return null; + } + + private static String extractResponseText(Map response) { + if (response == null) return null; + Object resp = response.get("response"); + if (resp instanceof String s && !s.isBlank()) return s; + Object output = response.get("output"); + if (output instanceof String s && !s.isBlank()) return s; + Object content = response.get("content"); + if (content instanceof String s && !s.isBlank()) return s; + return null; + } + + private static String stripCodeFences(String raw) { + String trimmed = raw.strip(); + if (trimmed.startsWith("```")) { + int firstNewline = trimmed.indexOf('\n'); + if (firstNewline != -1) trimmed = trimmed.substring(firstNewline + 1); + if (trimmed.endsWith("```")) trimmed = trimmed.substring(0, trimmed.lastIndexOf("```")).strip(); + } + return trimmed; + } + + @Override + public String getReactorDescription() { + return "Generates a SQL SELECT query from a plain-English description using the database schema and an AI model."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.DATABASE.getKey().equals(key)) return "ID of the database engine to query."; + if (AutomationConstants.DOC_DESCRIPTION.equals(key)) return "Plain-English description of what data to retrieve (base64-encoded)."; + if (ReactorKeysEnum.ENGINE.getKey().equals(key)) return "Optional model engine ID. Defaults to the first available MODEL engine."; + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/reactor/automation/GetReactorSignatureReactor.java b/src/prerna/reactor/automation/GetReactorSignatureReactor.java new file mode 100644 index 00000000000..667aee797ae --- /dev/null +++ b/src/prerna/reactor/automation/GetReactorSignatureReactor.java @@ -0,0 +1,227 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; + +import prerna.auth.User; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.project.api.IProject; +import prerna.reactor.AbstractReactor; +import prerna.reactor.IReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Returns a Pixel template string and description for a custom reactor in a project. + * + *

Pixel: {@code GetReactorSignature(project=["appId"], reactor=["ReactorName"])} + * + *

Returns a JSON object with: + *

    + *
  • {@code template} — a filled Pixel call showing each param placeholder, e.g. + * {@code MyReactor(requiredKey=[""], optionalKey="")}
    • + *
  • {@code description} — the reactor's one-line description, or empty string if none
    • + *
  • {@code hasParams} — boolean, false when the reactor declares no keys
    • + *
+ * + *

On any failure (project not found, reactor not found, bad metadata) returns a minimal + * fallback object with {@code template="ReactorName()"} so the caller can still populate the field. + */ +public final class GetReactorSignatureReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GetReactorSignatureReactor.class); + + private static final String REACTOR_KEY = "reactor"; + + public GetReactorSignatureReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), REACTOR_KEY }; + this.keyRequired = new int[] { 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } + + String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); + String reactorName = this.keyValue.get(REACTOR_KEY); + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access."); + } + if (reactorName == null || reactorName.trim().isEmpty()) { + throw new IllegalArgumentException("A reactor name is required."); + } + reactorName = reactorName.trim(); + + JSONObject result = buildSignature(projectId, reactorName); + return new NounMetadata(result.toString(), PixelDataType.CONST_STRING, PixelOperationType.OPERATION); + } + + private static JSONObject buildSignature(String projectId, String reactorName) { + JSONObject result = new JSONObject(); + result.put("reactorName", reactorName); + + IProject project = Utility.getProject(projectId); + if (project == null) { + classLogger.warn("GetReactorSignature: project {} not found", projectId); + return fallback(result, reactorName); + } + + IReactor reactor; + try { + reactor = project.getReactor(reactorName); + } catch (Exception e) { + classLogger.warn("GetReactorSignature: could not load reactor {} from project {}", reactorName, projectId, e); + return fallback(result, reactorName); + } + if (reactor == null) { + return fallback(result, reactorName); + } + + // Description — best-effort + String description = ""; + try { + String d = reactor.getReactorDescription(); + if (d != null && !d.isBlank()) description = d.trim(); + } catch (Exception e) { + classLogger.warn("GetReactorSignature: getReactorDescription() failed for {}", reactorName, e); + } + result.put("description", description); + + // Parameter metadata via asMcpTool() + try { + JSONObject tool = reactor.asMcpTool(); + JSONObject inputSchema = tool.optJSONObject("inputSchema"); + if (inputSchema == null) { + return fallback(result, reactorName); + } + + JSONObject properties = inputSchema.optJSONObject("properties"); + JSONArray required = inputSchema.optJSONArray("required"); + + if (properties == null || properties.isEmpty()) { + result.put("template", reactorName + "()"); + result.put("hasParams", false); + return result; + } + + // Build set of required key names for O(1) lookup + Set requiredKeys = new HashSet<>(); + if (required != null) { + for (int i = 0; i < required.length(); i++) { + requiredKeys.add(required.getString(i)); + } + } + + // Build per-param metadata and required-only template + StringBuilder template = new StringBuilder(reactorName).append("("); + JSONArray params = new JSONArray(); + boolean firstRequired = true; + + // Required params first (for template ordering) + for (String key : properties.keySet()) { + if (!requiredKeys.contains(key)) continue; + if (!firstRequired) template.append(", "); + template.append(key).append("=[\"\"]"); + firstRequired = false; + + JSONObject prop = properties.optJSONObject(key); + params.put(buildParamMeta(key, prop, true)); + } + // Then optional params (not in template, but included in params list) + for (String key : properties.keySet()) { + if (requiredKeys.contains(key)) continue; + JSONObject prop = properties.optJSONObject(key); + params.put(buildParamMeta(key, prop, false)); + } + template.append(")"); + + result.put("template", template.toString()); + result.put("hasParams", !properties.isEmpty()); + result.put("params", params); + } catch (Exception e) { + classLogger.warn("GetReactorSignature: asMcpTool() failed for {}", reactorName, e); + return fallback(result, reactorName); + } + + return result; + } + + private static JSONObject buildParamMeta(String key, JSONObject prop, boolean required) { + JSONObject meta = new JSONObject(); + meta.put("name", key); + meta.put("required", required); + if (prop != null) { + String type = prop.optString("type", "string"); + meta.put("type", type); + String desc = prop.optString("description", ""); + // Suppress the default placeholder description — it adds no value + if (!desc.isBlank() && !desc.equals("No description present")) { + meta.put("description", desc); + } + } else { + meta.put("type", "string"); + } + return meta; + } + + private static JSONObject fallback(JSONObject base, String reactorName) { + base.put("template", reactorName + "()"); + base.put("description", ""); + base.put("hasParams", false); + return base; + } + + @Override + public String getReactorDescription() { + return "Returns the Pixel call template and description for a custom reactor in a project."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (REACTOR_KEY.equals(key)) return "Name of the reactor to inspect (without 'Reactor' suffix)."; + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) return "Project ID containing the reactor."; + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/reactor/automation/TriggerAutomationReactor.java b/src/prerna/reactor/automation/TriggerAutomationReactor.java index 52a69469e7b..c028892c3db 100644 --- a/src/prerna/reactor/automation/TriggerAutomationReactor.java +++ b/src/prerna/reactor/automation/TriggerAutomationReactor.java @@ -69,6 +69,24 @@ public NounMetadata execute() { String userId = getUserId(); String runId = UUID.randomUUID().toString(); + // Validate the automation has runnable steps BEFORE claiming the run slot, + // so a bad automation never leaves a stale active-run record. + Map doc = AutomationExecutionUtils.loadAutomationDoc(projectId); + @SuppressWarnings("unchecked") + Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); + @SuppressWarnings("unchecked") + List> nodes = (List>) graph.get(AutomationConstants.DOC_NODES); + List> ordered = nodes != null ? nodes : new ArrayList<>(); + if (ordered.isEmpty()) { + throw new IllegalArgumentException("Automation has no nodes to execute"); + } + long nonTriggerCount = ordered.stream() + .filter(n -> !AutomationConstants.NODE_TRIGGER.equals(n.get(AutomationConstants.NODE_FIELD_TYPE))) + .count(); + if (nonTriggerCount == 0) { + throw new IllegalArgumentException("Automation has no steps to run. Add at least one step before running."); + } + if (!AutomationDatabaseUtility.claimActiveRun(projectId, runId)) { String activeRun = AutomationDatabaseUtility.getActiveRun(projectId); throw new IllegalArgumentException( @@ -78,18 +96,8 @@ public NounMetadata execute() { boolean runStarted = false; try { - Map doc = AutomationExecutionUtils.loadAutomationDoc(projectId); - @SuppressWarnings("unchecked") - Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); - @SuppressWarnings("unchecked") - List> nodes = (List>) graph.get(AutomationConstants.DOC_NODES); Map configMap = AutomationExecutionUtils.loadConfig(projectId); - List> ordered = nodes != null ? nodes : new ArrayList<>(); - if (ordered.isEmpty()) { - throw new IllegalArgumentException("Automation has no nodes to execute"); - } - @SuppressWarnings("unchecked") Map inputsMap = this.getMap(AutomationConstants.AUTOMATION_INPUTS_KEY); AutomationExecutionUtils.applyPlaygroundInputs(ordered, inputsMap); @@ -151,6 +159,7 @@ public NounMetadata execute() { ? AutomationExecutionUtils.buildSummaryMessage(doc, finalScope, configMap, completedCount, ordered.size()) : buildFailureSummary(runDetail); runDetail.put(AutomationConstants.RESULT_SUMMARY, summary); + AutomationDatabaseUtility.updateRunSummary(runId, summary); // Enriched context sent to the LLM as the MCP tool response, so it can describe // what each step actually did rather than just echoing the node count. diff --git a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java index 246f125b82b..0d53c406fab 100644 --- a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java +++ b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java @@ -292,7 +292,8 @@ public void createColumnsAndTypes() { Pair.with(AutomationConstants.TOTAL_NODES, INTEGER), Pair.with(AutomationConstants.COMPLETED_NODES, INTEGER), Pair.with(AutomationConstants.CREATED_BY, VARCHAR_255), - Pair.with(AutomationConstants.CANCEL_REQUESTED, BOOLEAN))); + Pair.with(AutomationConstants.CANCEL_REQUESTED, BOOLEAN), + Pair.with(AutomationConstants.RESULT_SUMMARY_COL, VARCHAR_2000))); addTable(AutomationConstants.TABLE_AUTOMATION_NODE_OUTPUTS, Arrays.asList( Pair.with(AutomationConstants.RUN_ID, VARCHAR_255), From 935eeaa5f1bbc75595c27eb0a330cd11cd17125c Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Thu, 6 Aug 2026 12:00:44 -0400 Subject: [PATCH 17/25] fix: squash bug --- src/prerna/reactor/automation/AutomationRunEngine.java | 2 +- .../reactor/automation/GenerateAutomationReactor.java | 9 +++++++-- .../reactor/automation/GetAutomationRunReactor.java | 6 +++++- .../reactor/automation/RunAutomationNodeReactor.java | 4 +--- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/prerna/reactor/automation/AutomationRunEngine.java b/src/prerna/reactor/automation/AutomationRunEngine.java index 4ff736cf693..848fb0c957a 100644 --- a/src/prerna/reactor/automation/AutomationRunEngine.java +++ b/src/prerna/reactor/automation/AutomationRunEngine.java @@ -248,7 +248,7 @@ private static Map executeSingleNode(String runId, String projec classLogger.debug("Node {} ({}) succeeded in {}ms in run {}", nodeId, nodeLabel, durationMs, runId); Map result = buildNodeResult(nodeId, nodeLabel, - AutomationConstants.NODE_STATUS_SUCCESS, durationMs, preview, null); + AutomationConstants.NODE_STATUS_SUCCESS, durationMs, transformed, null); result.put(AutomationConstants.RESULT_OUTPUT_VALUE, transformed); return result; diff --git a/src/prerna/reactor/automation/GenerateAutomationReactor.java b/src/prerna/reactor/automation/GenerateAutomationReactor.java index ff619c35e90..4bbae1f4c1a 100644 --- a/src/prerna/reactor/automation/GenerateAutomationReactor.java +++ b/src/prerna/reactor/automation/GenerateAutomationReactor.java @@ -98,7 +98,9 @@ public final class GenerateAutomationReactor extends AbstractReactor { - wait: config={"seconds":"5"} — pause between steps, outputVar="wait_out" ## Variable substitution -Reference upstream node outputs in config fields using ${outputVar}. +Reference upstream node outputs in config fields using ${outputVar} (e.g. ${db_out}, ${model_out}). +NEVER use SQL parameterized syntax ($1, $2, ?, :param) — those are not supported and will cause runtime errors. +In SQL expressions, always wrap ${outputVar} in single quotes for string/UUID values: WHERE col = '${varName}'. NEVER use double quotes — PostgreSQL treats double-quoted values as column names. ## Rules 1. Always start with a trigger node (id="trigger-1"). @@ -137,7 +139,9 @@ public final class GenerateAutomationReactor extends AbstractReactor { - wait: config={"seconds":"5"} — pause between steps, outputVar="wait_out" ## Variable substitution -Reference upstream node outputs in config fields using ${outputVar}. +Reference upstream node outputs in config fields using ${outputVar} (e.g. ${db_out}, ${model_out}). +NEVER use SQL parameterized syntax ($1, $2, ?, :param) — those are not supported and will cause runtime errors. +In SQL expressions, always wrap ${outputVar} in single quotes for string/UUID values: WHERE col = '${varName}'. NEVER use double quotes — PostgreSQL treats double-quoted values as column names. ## Rules 1. Always start with a trigger node (id="trigger-1"). @@ -168,6 +172,7 @@ public final class GenerateAutomationReactor extends AbstractReactor { - Do not add conditions or filters that aren't implied by the user's request - Use column types to write type-safe SQL — don't compare VARCHAR columns to integers or assume a column's semantics from its name alone - Use a reasonable LIMIT to avoid returning unbounded result sets + - NEVER use SQL parameterized placeholders ($1, $2, ?, :param) — the execution engine does not support bound parameters. For runtime values from upstream nodes, use ${outputVar} inline in the SQL string wrapped in single quotes (e.g. WHERE id = '${db_out}'). NEVER wrap ${outputVar} in double quotes — double quotes are SQL identifier delimiters and will cause a "column does not exist" error. For literal filters, hardcode the value directly. - If no relevant table exists in the schema, set expression to "-- [replace with your SQL query]" and update the label to "Review: update this query" 2. model-engine nodes: if the "command" references an upstream database node's output, update it to mention the specific column names the SQL query will actually return. 3. app nodes: replace the "pixel" value "PENDING_PIXEL_EXPRESSION" with a call to the most appropriate reactor from the available reactors list. diff --git a/src/prerna/reactor/automation/GetAutomationRunReactor.java b/src/prerna/reactor/automation/GetAutomationRunReactor.java index 85bf87e9fc6..32695ee7997 100644 --- a/src/prerna/reactor/automation/GetAutomationRunReactor.java +++ b/src/prerna/reactor/automation/GetAutomationRunReactor.java @@ -94,7 +94,11 @@ public NounMetadata execute() { nodeResult.put(AutomationConstants.NODE_LABEL, nodeOutput.get(AutomationConstants.NODE_LABEL)); nodeResult.put(AutomationConstants.STATUS, nodeOutput.get(AutomationConstants.STATUS)); nodeResult.put(AutomationConstants.DURATION_MS, nodeOutput.get(AutomationConstants.DURATION_MS)); - nodeResult.put(AutomationConstants.OUTPUT_PREVIEW, nodeOutput.get(AutomationConstants.OUTPUT_PREVIEW)); + String outputForDisplay = (String) nodeOutput.get(AutomationConstants.OUTPUT_VALUE); + if (outputForDisplay == null || outputForDisplay.isBlank()) { + outputForDisplay = (String) nodeOutput.get(AutomationConstants.OUTPUT_PREVIEW); + } + nodeResult.put(AutomationConstants.OUTPUT_PREVIEW, outputForDisplay); nodeResult.put(AutomationConstants.ERROR_MESSAGE, nodeOutput.get(AutomationConstants.ERROR_MESSAGE)); nodeResults.add(nodeResult); } diff --git a/src/prerna/reactor/automation/RunAutomationNodeReactor.java b/src/prerna/reactor/automation/RunAutomationNodeReactor.java index 1951dd8b4ee..8f8efffcb03 100644 --- a/src/prerna/reactor/automation/RunAutomationNodeReactor.java +++ b/src/prerna/reactor/automation/RunAutomationNodeReactor.java @@ -115,13 +115,11 @@ public NounMetadata execute() { Map transformConfig = (Map) node.get(AutomationConstants.NODE_FIELD_OUTPUT_TRANSFORM); String transformed = AutomationExecutionUtils.applyOutputTransform(rawOutput, transformConfig); long durationMs = System.currentTimeMillis() - startMs; - String preview = AutomationExecutionUtils.generatePreview(transformed); - Map result = new HashMap<>(); result.put(AutomationConstants.NODE_ID, nodeId); result.put(AutomationConstants.STATUS, AutomationConstants.NODE_STATUS_SUCCESS); result.put(AutomationConstants.DURATION_MS, durationMs); - result.put(AutomationConstants.OUTPUT_PREVIEW, preview); + result.put(AutomationConstants.OUTPUT_PREVIEW, transformed); result.put(AutomationConstants.OUTPUT_VALUE, transformed); return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); From 6eee44519af87cada8295433b9a5171c1e07ce1b Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Thu, 6 Aug 2026 13:59:42 -0400 Subject: [PATCH 18/25] feat: update prompts --- .../reactor/automation/GenerateAutomationReactor.java | 6 +++--- src/prerna/reactor/automation/TriggerAutomationReactor.java | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/prerna/reactor/automation/GenerateAutomationReactor.java b/src/prerna/reactor/automation/GenerateAutomationReactor.java index 4bbae1f4c1a..72bf02fdaaf 100644 --- a/src/prerna/reactor/automation/GenerateAutomationReactor.java +++ b/src/prerna/reactor/automation/GenerateAutomationReactor.java @@ -111,7 +111,7 @@ NEVER use SQL parameterized syntax ($1, $2, ?, :param) — those are not support 6. Build a realistic, useful graph — don't add unnecessary nodes. 7. For database-engine nodes: set expression to exactly "PENDING_SQL_GENERATION" — a second pass will fill in real SQL using the actual schema. 8. For app nodes: set pixel to exactly "PENDING_PIXEL_EXPRESSION" — a second pass will fill in the reactor call using the project's available reactors. -9. For model-engine nodes: write a concrete instruction in "command" appropriate to the user's request, referencing upstream outputVars where relevant. Exact column names are not yet known; describe the intent clearly. +9. For model-engine nodes: "command" is the plain instruction to the LLM (e.g. "Summarize these cases and highlight urgent items"). Put the actual data by setting "context" to the upstream outputVar (e.g. "context":"${db_out}"). NEVER describe the data structure in prose inside "command" — the LLM will receive the real data at runtime via ${outputVar} substitution, not a description of it. 10. Respond with ONLY valid JSON. No markdown, no code fences, no explanation. ## Response format @@ -152,7 +152,7 @@ NEVER use SQL parameterized syntax ($1, $2, ?, :param) — those are not support 6. Build a realistic, useful graph — don't add unnecessary nodes. 7. For database-engine nodes: set expression to exactly "PENDING_SQL_GENERATION" — a second pass will fill in real SQL using the actual schema. 8. For app nodes: set pixel to exactly "PENDING_PIXEL_EXPRESSION" — a second pass will fill in the reactor call using the project's available reactors. -9. For model-engine nodes: write a concrete instruction in "command" appropriate to the user's request, referencing upstream outputVars where relevant. Exact column names are not yet known; describe the intent clearly. +9. For model-engine nodes: "command" is the plain instruction to the LLM (e.g. "Summarize these cases and highlight urgent items"). Put the actual data by setting "context" to the upstream outputVar (e.g. "context":"${db_out}"). NEVER describe the data structure in prose inside "command" — the LLM will receive the real data at runtime via ${outputVar} substitution, not a description of it. 10. Respond with ONLY valid JSON. No markdown, no code fences, no explanation. ## Response format @@ -174,7 +174,7 @@ NEVER use SQL parameterized syntax ($1, $2, ?, :param) — those are not support - Use a reasonable LIMIT to avoid returning unbounded result sets - NEVER use SQL parameterized placeholders ($1, $2, ?, :param) — the execution engine does not support bound parameters. For runtime values from upstream nodes, use ${outputVar} inline in the SQL string wrapped in single quotes (e.g. WHERE id = '${db_out}'). NEVER wrap ${outputVar} in double quotes — double quotes are SQL identifier delimiters and will cause a "column does not exist" error. For literal filters, hardcode the value directly. - If no relevant table exists in the schema, set expression to "-- [replace with your SQL query]" and update the label to "Review: update this query" -2. model-engine nodes: if the "command" references an upstream database node's output, update it to mention the specific column names the SQL query will actually return. +2. model-engine nodes: ensure the "context" field contains the upstream outputVar reference (e.g. "${db_out}") so the actual data is passed to the LLM at runtime. The "command" must be a plain instruction only — NEVER describe the data structure or column names in prose inside "command". The LLM will see the real data via the context field; it does not need to be told what columns exist. 3. app nodes: replace the "pixel" value "PENDING_PIXEL_EXPRESSION" with a call to the most appropriate reactor from the available reactors list. - Use the format: ReactorName(param="${varName}") referencing upstream outputVars where relevant - If no reactor in the list clearly matches the intent, set pixel to "-- [describe the Pixel expression to write here]" diff --git a/src/prerna/reactor/automation/TriggerAutomationReactor.java b/src/prerna/reactor/automation/TriggerAutomationReactor.java index c028892c3db..f898fe49fb2 100644 --- a/src/prerna/reactor/automation/TriggerAutomationReactor.java +++ b/src/prerna/reactor/automation/TriggerAutomationReactor.java @@ -130,7 +130,11 @@ public NounMetadata execute() { nodeResult.put(AutomationConstants.NODE_LABEL, output.get(AutomationConstants.NODE_LABEL)); nodeResult.put(AutomationConstants.STATUS, output.get(AutomationConstants.STATUS)); nodeResult.put(AutomationConstants.DURATION_MS, output.get(AutomationConstants.DURATION_MS)); - nodeResult.put(AutomationConstants.OUTPUT_PREVIEW, output.get(AutomationConstants.OUTPUT_PREVIEW)); + String outputForDisplay = (String) output.get(AutomationConstants.OUTPUT_VALUE); + if (outputForDisplay == null || outputForDisplay.isBlank()) { + outputForDisplay = (String) output.get(AutomationConstants.OUTPUT_PREVIEW); + } + nodeResult.put(AutomationConstants.OUTPUT_PREVIEW, outputForDisplay); nodeResult.put(AutomationConstants.OUTPUT_VALUE, output.get(AutomationConstants.OUTPUT_VALUE)); nodeResult.put(AutomationConstants.ERROR_MESSAGE, output.get(AutomationConstants.ERROR_MESSAGE)); nodeResults.add(nodeResult); From f520f1723394505b02990eb4c1267c6027219b9d Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Mon, 10 Aug 2026 14:53:23 -0400 Subject: [PATCH 19/25] feat: add create and edit automation reactors for mcp --- .../automation/AutomationExecutionUtils.java | 8 +- .../automation/CreateAutomationReactor.java | 159 +++++++++++ .../GetAutomationStructureReactor.java | 173 ++++++++++++ .../QuickEditAutomationReactor.java | 261 ++++++++++++++++++ src/prerna/util/Constants.java | 1 + src/prerna/util/SystemDefaultEngines.java | 2 +- 6 files changed, 601 insertions(+), 3 deletions(-) create mode 100644 src/prerna/reactor/automation/CreateAutomationReactor.java create mode 100644 src/prerna/reactor/automation/GetAutomationStructureReactor.java create mode 100644 src/prerna/reactor/automation/QuickEditAutomationReactor.java diff --git a/src/prerna/reactor/automation/AutomationExecutionUtils.java b/src/prerna/reactor/automation/AutomationExecutionUtils.java index 1a6d6615886..5d2c1022a2a 100644 --- a/src/prerna/reactor/automation/AutomationExecutionUtils.java +++ b/src/prerna/reactor/automation/AutomationExecutionUtils.java @@ -29,6 +29,7 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.Type; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.time.Instant; @@ -69,6 +70,9 @@ public final class AutomationExecutionUtils { /** Prefix for config-map lookups within a template, e.g. {@code ${config.API_KEY}}. */ private static final String CONFIG_VAR_PREFIX = "config."; + /** Reusable {@link TypeToken} type for {@code Map} deserialization. */ + public static final Type MAP_TYPE = new TypeToken>() {}.getType(); + /** * Shared Gson instance for the whole automation engine — public so the * {@code nodes} sub-package has one shared instance to reuse instead of each @@ -310,7 +314,7 @@ private static Object parseJsonAny(String json) { private static Map parseJson(String json) { if (json == null || json.isBlank()) return null; try { - return GSON.fromJson(json, new TypeToken>() {}.getType()); + return GSON.fromJson(json, MAP_TYPE); } catch (Exception e) { return null; } @@ -481,7 +485,7 @@ public static Map loadAutomationDoc(String projectId) { } try { String json = Files.readString(f.toPath(), StandardCharsets.UTF_8); - return GSON.fromJson(json, new TypeToken>() {}.getType()); + return GSON.fromJson(json, MAP_TYPE); } catch (IOException e) { throw new IllegalStateException("Failed to read automation.json: " + e.getMessage(), e); } diff --git a/src/prerna/reactor/automation/CreateAutomationReactor.java b/src/prerna/reactor/automation/CreateAutomationReactor.java new file mode 100644 index 00000000000..69208d6504e --- /dev/null +++ b/src/prerna/reactor/automation/CreateAutomationReactor.java @@ -0,0 +1,159 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; + +/** + * Creates a new automation project and returns its ID so the LLM can immediately chain to + * {@code EditAutomation} to interactively build it. + * + *

The project name must start with a letter and contain only letters, numbers, and spaces + * (enforced by {@code CreateProject}). + * + *

Typical LLM flow: + *

    + *
  1. LLM calls {@code CreateAutomation(projectName=["My Automation"])} — auto, no UI
    2. + *
  2. LLM immediately chains {@code EditAutomation(project=[""], instruction=["..."])}
    3. + *
  3. User sees the editor open with the AI-generated draft
    4. + *
+ * + *

Pixel: {@code CreateAutomation(projectName=["My Claims Intake"])} + */ +public class CreateAutomationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(CreateAutomationReactor.class); + + private static final String PROJECT_NAME_KEY = "projectName"; + + private static final String RESULT_SUCCESS = "success"; + private static final String RESULT_PROJECT_ID = "projectId"; + private static final String RESULT_PROJECT_NAME = "projectName"; + private static final String RESULT_MESSAGE = "message"; + + public CreateAutomationReactor() { + this.keysToGet = new String[] { PROJECT_NAME_KEY }; + this.keyRequired = new int[] { 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectName = this.keyValue.get(PROJECT_NAME_KEY); + + if (projectName == null || projectName.trim().isEmpty()) { + throw new IllegalArgumentException("Must provide a projectName"); + } + projectName = projectName.trim(); + + classLogger.info("CreateAutomationReactor: creating project '{}'", projectName); + + // Validate before injecting into the pixel string — only letters, numbers, and spaces; must start + // with a letter. CreateProject enforces this too, but we reject here to prevent pixel injection. + if (!projectName.matches("^[a-zA-Z][a-zA-Z0-9 ]*$")) { + throw new IllegalArgumentException( + "Project name must start with a letter and contain only letters, numbers, and spaces. Got: " + projectName); + } + + // Delegate to CreateProject — it handles project scaffolding and security. + // CODE project type matches all existing automation projects. + String createPixel = String.format( + "CreateProject(project=[\"%s\"], projectType=[\"CODE\"], global=[false]);", + projectName); + + Object raw; + try { + raw = PixelExecutionUtils.runAndCollect(this.insight, createPixel); + } catch (PixelExecutionUtils.AutomationPixelException e) { + classLogger.error("CreateProject pixel error for '{}'", projectName, e); + throw new IllegalArgumentException("Failed to create project '" + projectName + "': " + e.getMessage()); + } + + if (!(raw instanceof Map)) { + classLogger.error("CreateProject returned unexpected result type for '{}': {}", + projectName, raw == null ? "null" : raw.getClass().getName()); + throw new IllegalArgumentException("Unexpected response from CreateProject for: " + projectName); + } + + @SuppressWarnings("unchecked") + Map projectData = (Map) raw; + String projectId = (String) projectData.get("project_id"); + + if (projectId == null || projectId.isBlank()) { + classLogger.error("CreateProject did not return a project_id for '{}'", projectName); + throw new IllegalArgumentException("Project was created but no project ID was returned for: " + projectName); + } + + classLogger.info("CreateAutomationReactor: created project '{}' with id {}", projectName, projectId); + + Map result = new LinkedHashMap<>(); + result.put(RESULT_SUCCESS, true); + result.put(RESULT_PROJECT_ID, projectId); + result.put(RESULT_PROJECT_NAME, projectName); + result.put(RESULT_MESSAGE, + "Created automation project \"" + projectName + "\" (id: " + projectId + "). " + + "Call EditAutomation(project=[\"" + projectId + "\"], instruction=[\"\"]) " + + "to open the editor and build the automation."); + + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + return meta; + } + + @Override + public String getReactorDescription() { + return "Creates a new blank automation project and returns its ID. " + + "Immediately chain EditAutomation with the returned project ID to interactively build the automation. " + + "Project names must start with a letter and contain only letters, numbers, and spaces."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (PROJECT_NAME_KEY.equals(key)) { + return "Display name for the new automation project. " + + "Must start with a letter and contain only letters, numbers, and spaces."; + } + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/reactor/automation/GetAutomationStructureReactor.java b/src/prerna/reactor/automation/GetAutomationStructureReactor.java new file mode 100644 index 00000000000..8821f56052a --- /dev/null +++ b/src/prerna/reactor/automation/GetAutomationStructureReactor.java @@ -0,0 +1,173 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.reflect.TypeToken; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.AssetUtility; + +/** + * Returns a compact summary of the automation's current structure — description and a flat + * node list (label, type, operation, engineId) — for use as LLM context before editing. + * + *

Trigger nodes are excluded; they are infrastructure, not user-authored steps. + * Returns {@code { description: "", nodes: [] }} for a blank or missing automation. + * + *

Pixel: {@code GetAutomationStructure(project=["appId"])} + */ +public class GetAutomationStructureReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GetAutomationStructureReactor.class); + + private static final String RESULT_KEY_DESCRIPTION = "description"; + private static final String RESULT_KEY_NODES = "nodes"; + + public GetAutomationStructureReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey() }; + this.keyRequired = new int[] { 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + + Map result = new LinkedHashMap<>(); + result.put(RESULT_KEY_DESCRIPTION, ""); + result.put(RESULT_KEY_NODES, new ArrayList<>()); + + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File automationFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); + + if (!automationFile.exists() || !automationFile.isFile()) { + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + try { + String json = Files.readString(automationFile.toPath(), StandardCharsets.UTF_8); + Map doc = AutomationExecutionUtils.GSON.fromJson(json, + new TypeToken>() {}.getType()); + + String description = (String) doc.getOrDefault(AutomationConstants.DOC_DESCRIPTION, ""); + result.put(RESULT_KEY_DESCRIPTION, description != null ? description : ""); + + Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); + if (graph != null) { + List nodes = (List) graph.get(AutomationConstants.DOC_NODES); + if (nodes != null) { + result.put(RESULT_KEY_NODES, buildNodeSummary(nodes)); + } + } + } catch (IOException e) { + classLogger.error("Error reading automation.json for project {}", projectId, e); + throw new IllegalArgumentException("Unable to read automation: " + e.getMessage()); + } + + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + /** + * Extracts a compact summary for each non-trigger node. + * Preserves label, type, and the two most useful config fields for LLM context. + */ + @SuppressWarnings("unchecked") + private static List> buildNodeSummary(List nodes) { + List> summary = new ArrayList<>(); + for (Object raw : nodes) { + if (!(raw instanceof Map)) { + continue; + } + Map node = (Map) raw; + String type = (String) node.get(AutomationConstants.NODE_FIELD_TYPE); + + // Trigger nodes are not user-authored steps — skip them + if (AutomationConstants.NODE_TRIGGER.equals(type)) { + continue; + } + + Map entry = new LinkedHashMap<>(); + entry.put(AutomationConstants.NODE_FIELD_LABEL, + node.getOrDefault(AutomationConstants.NODE_FIELD_LABEL, AutomationConstants.UNNAMED_NODE_LABEL)); + entry.put(AutomationConstants.NODE_FIELD_TYPE, type != null ? type : ""); + + Map config = (Map) node.get(AutomationConstants.NODE_FIELD_CONFIG); + if (config != null) { + if (config.containsKey(AutomationConstants.CONFIG_OPERATION)) { + entry.put(AutomationConstants.CONFIG_OPERATION, config.get(AutomationConstants.CONFIG_OPERATION)); + } + if (config.containsKey(AutomationConstants.CONFIG_ENGINE_ID)) { + entry.put(AutomationConstants.CONFIG_ENGINE_ID, config.get(AutomationConstants.CONFIG_ENGINE_ID)); + } + } + + summary.add(entry); + } + return summary; + } + + @Override + public String getReactorDescription() { + return "Returns the automation's description and a compact node list (label, type, operation, engineId) " + + "for LLM context. Trigger nodes are excluded. Returns an empty node list for a blank automation."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) { + return "The project ID of the automation to inspect."; + } + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/reactor/automation/QuickEditAutomationReactor.java b/src/prerna/reactor/automation/QuickEditAutomationReactor.java new file mode 100644 index 00000000000..6a2b4a040d7 --- /dev/null +++ b/src/prerna/reactor/automation/QuickEditAutomationReactor.java @@ -0,0 +1,261 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Base64; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.AssetUtility; + +/** + * Headless automation editor. The LLM provides a plain-English description of the desired change; + * this reactor chains {@code GenerateAutomation} (edit mode) and {@code SaveAutomation} silently + * without opening any UI. Returns a compact summary so the LLM can narrate the result. + * + *

Use {@code EditAutomation} instead when the change is complex or when the user wants to + * review the result before saving. + * + *

Threading: {@link PixelExecutionUtils#runAndCollect} blocks the caller thread for the full + * LLM round-trip (up to {@link AutomationConstants#DEFAULT_TIMEOUT_SECONDS} seconds). Acceptable + * for MVP usage volumes — revisit under concurrent load. + * + *

Pixel: {@code QuickEditAutomation(project=["appId"], editDescription=["change the SQL to last 7 days"])} + */ +public class QuickEditAutomationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(QuickEditAutomationReactor.class); + + private static final String EDIT_DESCRIPTION_KEY = "editDescription"; + + private static final String RESULT_SUCCESS = "success"; + private static final String RESULT_NODE_COUNT = "nodeCount"; + private static final String RESULT_MESSAGE = "message"; + + public QuickEditAutomationReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), EDIT_DESCRIPTION_KEY }; + this.keyRequired = new int[] { 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + String editDescription = this.keyValue.get(EDIT_DESCRIPTION_KEY); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + if (editDescription == null || editDescription.trim().isEmpty()) { + throw new IllegalArgumentException("Must provide an editDescription"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have edit access"); + } + + classLogger.info("QuickEditAutomationReactor: starting edit for project {}", projectId); + + String currentDocJson = loadCurrentDoc(projectId); + + // GenerateAutomation decodes base64 to prevent Pixel injection — same pattern used by the FE + String encodedDesc = Base64.getEncoder() + .encodeToString(editDescription.trim().getBytes(StandardCharsets.UTF_8)); + String encodedDoc = Base64.getEncoder() + .encodeToString(currentDocJson.getBytes(StandardCharsets.UTF_8)); + + String generatePixel = String.format( + "GenerateAutomation(project=[\"%s\"], description=[\"%s\"], currentDoc=[\"%s\"]);", + projectId, encodedDesc, encodedDoc); + + classLogger.info("QuickEditAutomationReactor: calling GenerateAutomation for project {}", projectId); + Object raw; + try { + raw = PixelExecutionUtils.runAndCollect(this.insight, generatePixel); + } catch (PixelExecutionUtils.AutomationPixelException e) { + classLogger.error("GenerateAutomation pixel error for project {}", projectId, e); + throw new IllegalArgumentException("AI generation failed: " + e.getMessage()); + } + + if (raw == null) { + throw new IllegalArgumentException("GenerateAutomation returned no result for project: " + projectId); + } + String generatedJson = raw instanceof String ? (String) raw : raw.toString(); + if (generatedJson.isBlank()) { + throw new IllegalArgumentException("GenerateAutomation returned an empty result for project: " + projectId); + } + + Map generatedDoc = parseAndValidate(generatedJson, projectId); + + String encodedSave = Base64.getEncoder() + .encodeToString(generatedJson.getBytes(StandardCharsets.UTF_8)); + String savePixel = String.format( + "SaveAutomation(project=[\"%s\"], json=[\"%s\"]);", + projectId, encodedSave); + + classLogger.info("QuickEditAutomationReactor: calling SaveAutomation for project {}", projectId); + try { + PixelExecutionUtils.runAndCollect(this.insight, savePixel); + } catch (PixelExecutionUtils.AutomationPixelException e) { + classLogger.error("SaveAutomation pixel error for project {}", projectId, e); + throw new IllegalArgumentException("Failed to save automation: " + e.getMessage()); + } + + Map summary = buildSummary(generatedDoc); + classLogger.info("QuickEditAutomationReactor: completed for project {}", projectId); + return new NounMetadata(summary, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + /** + * Reads the current automation.json from disk. + * Returns a minimal blank document if none exists — GenerateAutomation treats the absence of + * meaningful nodes as a fresh-start signal. + */ + private String loadCurrentDoc(String projectId) { + try { + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File automationFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); + if (automationFile.exists() && automationFile.isFile()) { + return Files.readString(automationFile.toPath(), StandardCharsets.UTF_8); + } + } catch (IOException e) { + classLogger.warn("Could not read current automation.json for project {} — treating as blank", projectId, e); + } + return "{\"version\":1,\"graph\":{\"nodes\":[],\"edges\":[]}}"; + } + + /** + * Parses the generated JSON and validates its structure before allowing a save. + * Throws if the document is malformed or missing the trigger node — prevents corrupt saves. + */ + @SuppressWarnings("unchecked") + private Map parseAndValidate(String json, String projectId) { + Map doc; + try { + doc = AutomationExecutionUtils.GSON.fromJson(json, AutomationExecutionUtils.MAP_TYPE); + } catch (Exception e) { + classLogger.error("GenerateAutomation returned invalid JSON for project {}", projectId, e); + throw new IllegalArgumentException("AI generation returned invalid JSON — not saved."); + } + + Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); + if (graph == null) { + classLogger.error("GenerateAutomation result missing 'graph' field for project {}", projectId); + throw new IllegalArgumentException("AI generation result is missing 'graph' field — not saved."); + } + + List nodes = (List) graph.get(AutomationConstants.DOC_NODES); + if (nodes == null || nodes.isEmpty()) { + classLogger.error("GenerateAutomation result has no nodes for project {}", projectId); + throw new IllegalArgumentException("AI generation result has no nodes — not saved."); + } + + boolean hasTrigger = nodes.stream() + .filter(n -> n instanceof Map) + .map(n -> (Map) n) + .anyMatch(n -> AutomationConstants.NODE_TRIGGER.equals(n.get(AutomationConstants.NODE_FIELD_TYPE))); + + if (!hasTrigger) { + classLogger.error("GenerateAutomation result has no trigger node for project {}", projectId); + throw new IllegalArgumentException("AI generation result has no trigger node — not saved."); + } + + return doc; + } + + /** + * Builds the summary Map returned to the LLM — concise enough to fit in context without + * sending the full automation JSON. + */ + @SuppressWarnings("unchecked") + private static Map buildSummary(Map doc) { + Map summary = new LinkedHashMap<>(); + summary.put(RESULT_SUCCESS, true); + + try { + String desc = (String) doc.getOrDefault(AutomationConstants.DOC_DESCRIPTION, ""); + Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); + List nodes = graph != null ? (List) graph.get(AutomationConstants.DOC_NODES) : null; + int nodeCount = nodes != null ? nodes.size() : 0; + summary.put(RESULT_NODE_COUNT, nodeCount); + + String message = (desc != null && !desc.isBlank()) + ? "Automation updated successfully. Description: \"" + desc.trim() + "\". " + nodeCount + " steps." + : "Automation updated successfully. " + nodeCount + " steps."; + summary.put(RESULT_MESSAGE, message); + } catch (Exception e) { + classLogger.warn("Could not build summary from generated doc", e); + summary.put(RESULT_MESSAGE, "Automation updated successfully."); + } + + return summary; + } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + return meta; + } + + @Override + public String getReactorDescription() { + return "Modifies an automation using AI generation without opening any UI. " + + "Chains GenerateAutomation (edit mode) and SaveAutomation silently. " + + "Use EditAutomation instead when the user needs to review the change before saving."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) { + return "The project ID of the automation to edit."; + } else if (EDIT_DESCRIPTION_KEY.equals(key)) { + return "Plain-language description of the change to make. " + + "Example: 'Change the SQL filter to pull records from the last 7 days'. " + + "Call GetAutomationStructure first to understand the current automation structure."; + } + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/util/Constants.java b/src/prerna/util/Constants.java index 7ac02183028..fa60404d638 100644 --- a/src/prerna/util/Constants.java +++ b/src/prerna/util/Constants.java @@ -1083,6 +1083,7 @@ public class Constants { public static final String MCP_DATABASE_MAKER = "database-maker"; public static final String MCP_REACTOR_HELP = "reactor-help"; public static final String MCP_BROWSER_AUTOMATION = "browser-automation"; + public static final String MCP_AUTOMATION = "automation-portal"; // system (platform) agent (workspace) names public static final String AGENT_APP_BUILDER = "app-builder"; diff --git a/src/prerna/util/SystemDefaultEngines.java b/src/prerna/util/SystemDefaultEngines.java index 17060d6e8ab..11db564b4b7 100644 --- a/src/prerna/util/SystemDefaultEngines.java +++ b/src/prerna/util/SystemDefaultEngines.java @@ -60,7 +60,7 @@ public class SystemDefaultEngines { * must have a matching {@code project/platform__} folder. */ private static final List SYSTEM_MCPS = List.of(Constants.MCP_NODE_BUILDER, Constants.MCP_DATABASE_MAKER, - Constants.MCP_REACTOR_HELP, Constants.MCP_BROWSER_AUTOMATION); + Constants.MCP_REACTOR_HELP, Constants.MCP_BROWSER_AUTOMATION, Constants.MCP_AUTOMATION); /** * Subset of {@link #SYSTEM_MCPS} seeded onto system agent workspaces. This is From 2ff2fb557f4ef378420f06384cce660ca1daad43 Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Thu, 13 Aug 2026 11:35:42 -0400 Subject: [PATCH 20/25] feat: automation improvements --- .../automation/AutomationAskRoomReactor.java | 200 +++++++++++ .../automation/AutomationConstants.java | 2 + .../automation/AutomationDatabaseUtility.java | 34 +- .../automation/AutomationExecutionUtils.java | 339 +++++++++++++++++- .../reactor/automation/AutomationMcpSync.java | 64 +++- .../automation/AutomationRunEngine.java | 2 +- .../automation/BuildAutomationReactor.java | 259 +++++++++++++ .../automation/ExplainAutomationReactor.java | 223 ++++++++++++ .../automation/GenerateAutomationReactor.java | 211 +---------- .../automation/GenerateNodeLabelReactor.java | 244 +++++++++++++ .../automation/GenerateRunSummaryReactor.java | 159 ++++++++ .../automation/GenerateSQLReactor.java | 244 ------------- .../automation/GetAutomationReactor.java | 13 +- .../SaveAutomationConfigReactor.java | 23 +- .../automation/SaveAutomationReactor.java | 21 +- .../nodes/AppEngineNodeExecutor.java | 17 +- .../nodes/DatabaseEngineNodeExecutor.java | 27 +- .../nodes/FunctionEngineNodeExecutor.java | 16 +- .../nodes/ModelEngineNodeExecutor.java | 38 +- .../automation/nodes/NodeConfigHelper.java | 100 ++++++ .../nodes/StorageEngineNodeExecutor.java | 30 +- .../nodes/VectorEngineNodeExecutor.java | 31 +- .../automation/nodes/WaitNodeExecutor.java | 2 + 23 files changed, 1695 insertions(+), 604 deletions(-) create mode 100644 src/prerna/reactor/automation/AutomationAskRoomReactor.java create mode 100644 src/prerna/reactor/automation/BuildAutomationReactor.java create mode 100644 src/prerna/reactor/automation/ExplainAutomationReactor.java create mode 100644 src/prerna/reactor/automation/GenerateNodeLabelReactor.java create mode 100644 src/prerna/reactor/automation/GenerateRunSummaryReactor.java delete mode 100644 src/prerna/reactor/automation/GenerateSQLReactor.java create mode 100644 src/prerna/reactor/automation/nodes/NodeConfigHelper.java diff --git a/src/prerna/reactor/automation/AutomationAskRoomReactor.java b/src/prerna/reactor/automation/AutomationAskRoomReactor.java new file mode 100644 index 00000000000..11c0a4b5d8f --- /dev/null +++ b/src/prerna/reactor/automation/AutomationAskRoomReactor.java @@ -0,0 +1,200 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.engine.api.IModelEngine; +import prerna.engine.impl.model.RoomUtils; +import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.run.AgentRuntimeManager; +import prerna.reactor.agent.run.RunAgentRequest; +import prerna.reactor.agent.run.RunAgentResult; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Room-aware conversational AI assistant for building automation workflows. + * Uses the platform RunAgent harness for server-side history and MCP tool access. + * All user engines are registered as MCP tools so the model can query databases, + * search vectors, etc. during the design conversation. + * + *

Pixel: {@code AutomationAskRoom(project=["appId"], room=["roomId"], command=["base64text"])} + */ +public class AutomationAskRoomReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(AutomationAskRoomReactor.class); + + private static final String ROOM_KEY = "room"; + private static final String HARNESS_TYPE = "semoss"; + private static final int MAX_TURNS = 8; + private static final long CHAT_TIMEOUT_MS = 180_000L; + + private static final String SYSTEM_PROMPT = + "You are an AI assistant that helps users design automation workflows on a no-code platform. " + + "Workflows are linear sequences of steps: database queries, AI model calls, file storage, vector search, or custom functions. " + + "Only manual triggers exist — do not ask about scheduling.\n\n" + + "You have access to tools. Use them to help the user (for example, query a database to understand its structure " + + "so you can give accurate step descriptions).\n\n" + + "CONVERSATION PHASES:\n\n" + + "Phase 1 — Gather requirements (at most 1-2 short questions, one at a time):\n" + + "- Ask what the automation should do if unclear.\n" + + "- Ask where results should go, or one other essential clarification.\n" + + "- Keep responses under 50 words per question.\n\n" + + "Phase 2 — Plan + build signal (in ONE response, once you have enough info):\n" + + "- Present a concise numbered plain-English plan.\n" + + "- On the very next line after the plan, output the build signal JSON:\n" + + " {\"action\":\"build\",\"description\":\"\"}\n" + + "- The description must include all steps in enough detail for an AI to build them.\n" + + "- Do NOT ask 'does this look right?' — include the build signal in the same response as the plan.\n\n" + + "Phase 3 — If the user requests changes after seeing the plan:\n" + + "- Acknowledge in one short sentence.\n" + + "- Immediately output the revised plan + a new build signal in the same response.\n" + + "- Do NOT narrate what you will change — just show the revised plan and the signal.\n\n" + + "RULES: Never mention engine IDs, node types, or JSON structure to the user. Be concise."; + + public AutomationAskRoomReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), ROOM_KEY, ReactorKeysEnum.COMMAND.getKey() }; + this.keyRequired = new int[] { 1, 0, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } + + String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); + String roomId = this.keyValue.get(ROOM_KEY); + String rawCommand = this.keyValue.get(ReactorKeysEnum.COMMAND.getKey()); + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access."); + } + + String command = decodeCommand(rawCommand); + if (command == null || command.isBlank()) { + throw new IllegalArgumentException("command must not be empty."); + } + + String engineId = AutomationExecutionUtils.findFirstModelEngine(user); + if (engineId == null || engineId.isBlank()) { + throw new IllegalArgumentException( + "No AI model engine is available. Add a model engine connection to use this feature."); + } + + IModelEngine modelEngine = Utility.getModel(engineId); + if (modelEngine == null) { + throw new IllegalArgumentException("Model engine could not be loaded."); + } + + if (roomId == null || roomId.isBlank()) { + roomId = "automationchat" + projectId.replace("-", "").substring(0, Math.min(8, projectId.replace("-", "").length())); + } + + Map options = AutomationExecutionUtils.buildEngineMcpOptions(user, SYSTEM_PROMPT); + + RoomUtils.createRoomIfNotExists(roomId, this.insight, modelEngine, command, null, options, null, projectId, null); + + RunAgentRequest request = new RunAgentRequest( + roomId, command, engineId, HARNESS_TYPE, null, + MAX_TURNS, 0, null, null, null, null, this.insight); + + RunAgentResult handle = AgentRuntimeManager.get().run(request); + Map result; + try { + result = AgentRuntimeManager.get().waitForRun(handle.getRunId(), this.insight, CHAT_TIMEOUT_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Chat interrupted.", e); + } + + String status = (String) result.get("status"); + if ("FAILED".equals(status)) { + String errMsg = (String) result.get("errorMessage"); + classLogger.error("AutomationAskRoom run failed: project={} error={}", projectId, errMsg); + throw new RuntimeException("Chat failed: " + (errMsg != null ? errMsg : "unknown error")); + } + + String finalText = (String) result.get("finalText"); + if (finalText == null || finalText.isBlank()) { + throw new IllegalStateException("The AI model did not respond. Try again."); + } + + classLogger.info("AutomationAskRoom completed: project={}", projectId); + return new NounMetadata(finalText.strip(), PixelDataType.CONST_STRING, PixelOperationType.OPERATION); + } + + /** + * Decodes a base64-encoded command string. Falls back to the raw value when + * the input is not valid base64 (supports plain-text callers during testing). + */ + private static String decodeCommand(String raw) { + if (raw == null) { + return null; + } + try { + return new String(Base64.getDecoder().decode(raw.trim()), StandardCharsets.UTF_8); + } catch (Exception e) { + return raw; + } + } + + @Override + public String getReactorDescription() { + return "Room-aware conversational AI for designing automation workflows. " + + "Uses the platform RunAgent harness with MCP tool access — the model can query " + + "databases, search vectors, and use other engines during the conversation. " + + "History is managed server-side. " + + "Signals build-readiness via: {\"action\":\"build\",\"description\":\"...\"}"; + } + + @Override + protected String getDescriptionForKey(String key) { + return switch (key) { + case "project" -> "The project ID the automation belongs to."; + case "room" -> "Room ID for this conversation (defaults to automationchat{projectId})."; + case "command" -> "The user message, base64-encoded."; + default -> super.getDescriptionForKey(key); + }; + } +} diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index cc85392d151..1e9094b5ffc 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -250,4 +250,6 @@ private AutomationConstants() {} public static final int HEARTBEAT_INTERVAL_SECONDS = 30; public static final int STALE_HEARTBEAT_THRESHOLD_MINUTES = 5; public static final int OUTPUT_PREVIEW_MAX_LENGTH = 2000; + /** Maximum characters from a node output preview included in the AI run-summary prompt. */ + public static final int SUMMARY_PROMPT_PREVIEW_MAX_LENGTH = 300; } diff --git a/src/prerna/reactor/automation/AutomationDatabaseUtility.java b/src/prerna/reactor/automation/AutomationDatabaseUtility.java index 37df201ff7a..4f6c7305a9b 100644 --- a/src/prerna/reactor/automation/AutomationDatabaseUtility.java +++ b/src/prerna/reactor/automation/AutomationDatabaseUtility.java @@ -216,7 +216,7 @@ public static void initialize() { classLogger.info("Automation engine tables initialized successfully"); } catch (Exception e) { - classLogger.error("Failed to initialize automation engine tables: {}", e.getMessage(), e); + classLogger.error("Failed to initialize automation engine tables", e); } finally { closeConnection(schedulerDb, conn); } @@ -285,7 +285,7 @@ public static void markStaleRunsInterrupted() { conn.commit(); } } catch (Exception e) { - classLogger.error("Failed to mark stale automation runs: {}", e.getMessage(), e); + classLogger.error("Failed to mark stale automation runs", e); } finally { closeConnection(schedulerDb, conn); } @@ -388,8 +388,8 @@ public static boolean releaseActiveRun(String projectId, String runId) { } return true; } catch (SQLException e) { - classLogger.error("Failed to release active-run slot for project {}, run {}: {}", - projectId, runId, e.getMessage(), e); + classLogger.error("Failed to release active-run slot for project {}, run {}", + projectId, runId, e); return false; } finally { closeConnection(schedulerDb, conn); @@ -445,7 +445,7 @@ public static boolean setCancelRequested(String runId) { } return true; } catch (SQLException e) { - classLogger.error("Failed to set cancel-requested flag for run '{}': {}", runId, e.getMessage(), e); + classLogger.error("Failed to set cancel-requested flag for run '{}'", runId, e); return false; } finally { closeConnection(schedulerDb, conn); @@ -511,7 +511,7 @@ public static boolean insertRun(String runId, String projectId, String automatio } return true; } catch (SQLException e) { - classLogger.error("Failed to insert automation run '{}': {}", runId, e.getMessage(), e); + classLogger.error("Failed to insert automation run '{}'", runId, e); return false; } finally { closeConnection(schedulerDb, conn); @@ -543,7 +543,7 @@ public static boolean updateRunStatus(String runId, String status, } return true; } catch (SQLException e) { - classLogger.error("Failed to update run status for '{}': {}", runId, e.getMessage(), e); + classLogger.error("Failed to update run status for '{}'", runId, e); return false; } finally { closeConnection(schedulerDb, conn); @@ -572,7 +572,7 @@ public static boolean updateRunSummary(String runId, String resultSummary) { } return true; } catch (SQLException e) { - classLogger.error("Failed to update run summary for '{}': {}", runId, e.getMessage(), e); + classLogger.error("Failed to update run summary for '{}'", runId, e); return false; } finally { closeConnection(schedulerDb, conn); @@ -601,7 +601,7 @@ public static boolean updateHeartbeat(String runId, int completedNodes) { } return true; } catch (SQLException e) { - classLogger.error("Failed to update heartbeat for run '{}': {}", runId, e.getMessage(), e); + classLogger.error("Failed to update heartbeat for run '{}'", runId, e); return false; } finally { closeConnection(schedulerDb, conn); @@ -629,7 +629,7 @@ public static boolean touchHeartbeat(String runId) { } return true; } catch (SQLException e) { - classLogger.error("Failed to touch heartbeat for run '{}': {}", runId, e.getMessage(), e); + classLogger.error("Failed to touch heartbeat for run '{}'", runId, e); return false; } finally { closeConnection(schedulerDb, conn); @@ -734,7 +734,7 @@ public static boolean insertAllNodeOutputs(String runId, List} deserialization. */ public static final Type MAP_TYPE = new TypeToken>() {}.getType(); + /** Reusable {@link TypeToken} type for {@code List>} deserialization. */ + private static final Type LIST_OBJ_MAP_TYPE = new TypeToken>>() {}.getType(); + /** * Shared Gson instance for the whole automation engine — public so the * {@code nodes} sub-package has one shared instance to reuse instead of each @@ -145,11 +161,10 @@ public static Map loadConfig(String projectId) { Map map = new HashMap<>(); try { String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); - File f = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_CONFIG_FILE_NAME); + File f = Paths.get(portalsFolder, AutomationConstants.AUTOMATION_CONFIG_FILE_NAME).toFile(); if (!f.exists()) return map; String json = Files.readString(f.toPath(), StandardCharsets.UTF_8); - List> entries = GSON.fromJson(json, - new TypeToken>>() {}.getType()); + List> entries = GSON.fromJson(json, LIST_OBJ_MAP_TYPE); if (entries != null) { for (Map entry : entries) { String key = (String) entry.get(AutomationConstants.CONFIG_ENTRY_KEY); @@ -158,7 +173,7 @@ public static Map loadConfig(String projectId) { } } } catch (Exception e) { - classLogger.warn("Failed to load automation config for project {}: {}", projectId, e.getMessage(), e); + classLogger.warn("Failed to load automation config for project {}", projectId, e); } return map; } @@ -369,6 +384,57 @@ public static String buildSummaryMessage(Map doc, Map runDetail, + List> nodeOutputs) { + StringBuilder sb = new StringBuilder(); + + String status = strCfg(runDetail.get(AutomationConstants.STATUS)); + sb.append("Run status: ").append(status != null ? status : "unknown").append("\n"); + + String failedNodeId = strCfg(runDetail.get(AutomationConstants.FAILED_NODE_ID)); + String errorMsg = strCfg(runDetail.get(AutomationConstants.ERROR_MESSAGE)); + if (failedNodeId != null) { + sb.append("Failed at: ").append(failedNodeId).append("\n"); + } + if (errorMsg != null) { + sb.append("Error: ").append(errorMsg).append("\n"); + } + + if (!nodeOutputs.isEmpty()) { + sb.append("\nSteps:\n"); + for (Map node : nodeOutputs) { + String label = strCfg(node.get(AutomationConstants.NODE_LABEL)); + String nodeStatus = strCfg(node.get(AutomationConstants.STATUS)); + String preview = strCfg(node.get(AutomationConstants.OUTPUT_PREVIEW)); + String errDetail = strCfg(node.get(AutomationConstants.ERROR_MESSAGE)); + + sb.append("- ") + .append(label != null ? label : AutomationConstants.UNNAMED_NODE_LABEL) + .append(" [").append(nodeStatus != null ? nodeStatus : "unknown").append("]"); + + if (preview != null) { + String truncated = preview.length() > AutomationConstants.SUMMARY_PROMPT_PREVIEW_MAX_LENGTH + ? preview.substring(0, AutomationConstants.SUMMARY_PROMPT_PREVIEW_MAX_LENGTH) + "..." + : preview; + sb.append(": ").append(truncated); + } else if (errDetail != null) { + sb.append(": ").append(errDetail); + } + sb.append("\n"); + } + } + + return sb.toString(); + } + /** Truncates a string to {@link AutomationConstants#OUTPUT_PREVIEW_MAX_LENGTH} chars. */ public static String generatePreview(String s) { if (s == null) return null; @@ -470,6 +536,271 @@ public static Map coerceToMap(Object raw) { return new HashMap<>(); } + // -- LLM helpers (shared by GenerateAutomationReactor and ExplainAutomationReactor) --- + + /** + * Returns the ID of the first MODEL-type engine the user has access to, or {@code null} if none. + */ + public static String findFirstModelEngine(User user) { + try { + List> engines = SecurityEngineUtils.getUserEngineList( + user, List.of("MODEL"), null, false, null, null, null, "1", "0", null); + if (engines != null && !engines.isEmpty()) { + Object id = engines.get(0).get("database_id"); + return id != null ? String.valueOf(id) : null; + } + } catch (Exception e) { + classLogger.warn("Failed to auto-discover model engine", e); + } + return null; + } + + /** + * Extracts the text content from a model engine response map. + * Checks {@code response}, {@code output}, and {@code content} keys in order. + */ + public static String extractResponseText(Map response) { + if (response == null) return null; + Object resp = response.get("response"); + if (resp instanceof String s && !s.isBlank()) return s; + Object output = response.get("output"); + if (output instanceof String s && !s.isBlank()) return s; + Object content = response.get("content"); + if (content instanceof String s && !s.isBlank()) return s; + return null; + } + + // -- Generation helpers (shared by GenerateAutomationReactor and BuildAutomationReactor) --- + + /** + * Builds the Room options map for RunAgent — registers all user-accessible engines + * as MCP tools and includes the room's built-in tools. + */ + public static Map buildEngineMcpOptions(User user, String systemPrompt) { + List> mcpList = new ArrayList<>(); + try { + List> engines = SecurityEngineUtils.getUserEngineList( + user, GENERATION_ENGINE_TYPES, null, false, null, null, null, "50", "0", null); + if (engines != null) { + for (Map engine : engines) { + String id = engine.getOrDefault("database_id", "").toString(); + String name = engine.getOrDefault("database_name", "").toString(); + if (id.isBlank()) continue; + try { + IEngine engineObj = Utility.getEngine(id); + if (engineObj == null || !engineObj.isMCPEnabled()) continue; + } catch (Exception e) { + classLogger.debug("Skipping engine {} from MCP list — could not load: {}", id, e.getMessage()); + continue; + } + Map entry = new HashMap<>(); + entry.put("id", id); + entry.put("name", name); + entry.put("type", "ENGINE"); + mcpList.add(entry); + } + } + } catch (Exception e) { + classLogger.warn("Failed to build MCP engine list for automation room", e); + } + Map dbMakerEntry = new HashMap<>(); + dbMakerEntry.put("id", Constants.MCP_DATABASE_MAKER); + dbMakerEntry.put("name", "Database Tools"); + dbMakerEntry.put("type", "PROJECT"); + mcpList.add(dbMakerEntry); + Map roomEntry = new HashMap<>(); + roomEntry.put("id", MCPUtility.ROOM_MCP_ID); + roomEntry.put("name", MCPUtility.ROOM_MCP_NAME); + roomEntry.put("type", MCPUtility.ROOM_MCP_TYPE); + roomEntry.put("fromRoom", true); + mcpList.add(roomEntry); + Map options = new HashMap<>(); + if (systemPrompt != null && !systemPrompt.isBlank()) { + options.put("instructions", systemPrompt); + } + options.put("mcp", mcpList); + return options; + } + + /** Engine types listed in the generation prompt so the LLM can populate engineId fields. */ + static final List GENERATION_ENGINE_TYPES = Arrays.asList("DATABASE", "MODEL", "VECTOR", "STORAGE", "FUNCTION"); + + /** + * Builds the "## Available engines" prompt section for generation — lists each engine the + * current user can access so the LLM can assign engineId fields correctly. + */ + static String buildAvailableEnginesSection(User user) { + StringBuilder sb = new StringBuilder("## Available engines\n"); + try { + List> engines = SecurityEngineUtils.getUserEngineList( + user, GENERATION_ENGINE_TYPES, null, false, null, null, null, "50", "0", null); + if (engines == null || engines.isEmpty()) { + sb.append("None available — leave engineId as empty string.\n"); + return sb.toString(); + } + for (Map engine : engines) { + String id = engine.getOrDefault("database_id", "").toString(); + String name = engine.getOrDefault("database_name", "").toString(); + String type = engine.getOrDefault("engine_type", "").toString().toUpperCase(); + sb.append("- type=").append(type) + .append(" id=\"").append(id) + .append("\" name=\"").append(name).append("\""); + // For relational databases, include the table list so the model can match + // the user's intent to the right DB without needing to call get_db_schema first. + if ("DATABASE".equals(type) && !id.isBlank()) { + try { + IDatabaseEngine dbEngine = Utility.getDatabase(id); + if (dbEngine instanceof IRDBMSEngine rdbms) { + List tables = rdbms.getPixelConcepts(); + if (tables != null && !tables.isEmpty()) { + sb.append(" tables=["); + for (int i = 0; i < tables.size(); i++) { + String table = tables.get(i); + // Strip the "ConceptName__" prefix that pixel concepts sometimes include + int sep = table.lastIndexOf("__"); + sb.append(sep >= 0 ? table.substring(sep + 2) : table); + if (i < tables.size() - 1) sb.append(", "); + } + sb.append("]"); + } + } + } catch (Exception e) { + // Table list is best-effort; the model can still call get_db_schema if needed + } + } + sb.append("\n"); + } + } catch (Exception e) { + classLogger.warn("Failed to build engine list for generation prompt", e); + sb.append("(engine list unavailable — leave engineId as empty string)\n"); + } + return sb.toString(); + } + + /** + * Fetches the table/column schema (with data types) for the given list of database engine IDs. + * Formatted as a prompt section for LLM use. + */ + static String buildSchemaForEngineIds(List engineIds) { + StringBuilder sb = new StringBuilder("## Database schema\n"); + for (String engineId : engineIds) { + try { + IDatabaseEngine dbEngine = Utility.getDatabase(engineId); + if (!(dbEngine instanceof IRDBMSEngine rdbms)) { + sb.append("Database id=\"").append(engineId).append("\": (non-relational — no table schema)\n"); + continue; + } + sb.append("Database id=\"").append(engineId).append("\":\n"); + List tables = rdbms.getPixelConcepts(); + if (tables == null || tables.isEmpty()) { + sb.append(" (no tables found)\n"); + continue; + } + for (String table : tables) { + List columns = rdbms.getPixelSelectors(table); + sb.append(" - ").append(table).append(" ("); + if (columns != null && !columns.isEmpty()) { + StringBuilder cols = new StringBuilder(); + for (String col : columns) { + int sep = col.lastIndexOf("__"); + String colName = sep >= 0 ? col.substring(sep + 2) : col; + String dataType = null; + try { + String physicalUri = rdbms.getPhysicalUriFromPixelSelector(col); + if (physicalUri != null) { + dataType = rdbms.getDataTypes(physicalUri); + } + } catch (Exception ignored) { + // Data type fetch is best-effort; column name alone is still useful + } + if (cols.length() > 0) cols.append(", "); + cols.append(colName); + if (dataType != null && !dataType.isBlank()) { + cols.append(": ").append(dataType.trim()); + } + } + sb.append(cols); + } + sb.append(")\n"); + } + } catch (Exception e) { + classLogger.warn("Failed to fetch schema for engine {}", engineId, e); + sb.append("Database id=\"").append(engineId).append("\": (schema unavailable)\n"); + } + } + return sb.toString(); + } + + /** + * Returns the list of custom reactors available in the given project, formatted as a prompt + * section for LLM use. Used to fill in app-node pixel expressions. + */ + static String buildReactorListSection(String projectId) { + StringBuilder sb = new StringBuilder("## Available custom reactors in this project\n"); + try { + IProject project = Utility.getProject(projectId); + if (project == null) { + sb.append("(project not found — leave app node pixel as-is)\n"); + return sb.toString(); + } + java.util.TreeSet reactors = project.getAvailableReactors(); + if (reactors == null || reactors.isEmpty()) { + sb.append("(none — leave app node pixel as a comment placeholder)\n"); + return sb.toString(); + } + for (String name : reactors) { + sb.append("- ").append(name).append("\n"); + } + } catch (Exception e) { + classLogger.warn("Failed to build reactor list for project {}", projectId, e); + sb.append("(reactor list unavailable — leave app node pixel as a comment placeholder)\n"); + } + return sb.toString(); + } + + /** + * Strips leading/trailing markdown code fences ({@code ```json ... ```} or {@code ``` ... ```}) + * that models sometimes add despite being instructed not to. + */ + static String stripCodeFences(String raw) { + String trimmed = raw.strip(); + if (trimmed.startsWith("```")) { + int firstNewline = trimmed.indexOf('\n'); + if (firstNewline != -1) { + trimmed = trimmed.substring(firstNewline + 1); + } + if (trimmed.endsWith("```")) { + trimmed = trimmed.substring(0, trimmed.lastIndexOf("```")).strip(); + } + } + return trimmed; + } + + /** + * Validates that a generated document string is parseable JSON and contains the minimum + * required structure ({@code graph.nodes} must be non-empty). Throws on failure. + */ + static void validateGeneratedDoc(String raw) { + try { + JSONObject doc = new JSONObject(raw); + if (!doc.has("graph")) { + throw new IllegalStateException("Generated document is missing the 'graph' field."); + } + JSONObject graph = doc.getJSONObject("graph"); + if (!graph.has("nodes")) { + throw new IllegalStateException("Generated graph is missing the 'nodes' array."); + } + JSONArray nodes = graph.getJSONArray("nodes"); + if (nodes.length() == 0) { + throw new IllegalStateException("Generated graph has no nodes."); + } + } catch (org.json.JSONException e) { + classLogger.warn("Generated automation doc is not valid JSON (truncated): {}", raw.length() > 500 ? raw.substring(0, 500) + "..." : raw, e); + throw new IllegalStateException( + "The AI model returned an invalid response. Please try again with a different description.", e); + } + } + // -- Automation document loading ----------------------------------------------- /** diff --git a/src/prerna/reactor/automation/AutomationMcpSync.java b/src/prerna/reactor/automation/AutomationMcpSync.java index b09c1ea5994..e9c48843694 100644 --- a/src/prerna/reactor/automation/AutomationMcpSync.java +++ b/src/prerna/reactor/automation/AutomationMcpSync.java @@ -31,6 +31,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Paths; import java.time.LocalDate; import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; @@ -61,6 +62,10 @@ * *

A single-purpose class (not folded into {@link AutomationExecutionUtils}, which is scoped to * run-execution concerns) so the MCP-catalog-sync responsibility stays isolated and easy to find. + * + *

Uses {@code org.json} (JSONObject/JSONArray) throughout because {@link MCPUtility} is built + * on org.json, and converting between org.json and Gson just to call those helpers would add + * unnecessary overhead and type-safety risk. Gson is used everywhere else in the automation package. */ public final class AutomationMcpSync { @@ -98,6 +103,10 @@ public static void syncTriggerAutomationTool(IProject project, String projectId, projectId); return; } + if (automationJson == null || automationJson.isBlank()) { + classLogger.warn("Skipping automation MCP tool sync for project {}: automation JSON is empty.", projectId); + return; + } try { boolean hasDbNodes = hasPlaygroundDbNodes(automationJson); @@ -105,10 +114,11 @@ public static void syncTriggerAutomationTool(IProject project, String projectId, if (hasDbNodes) { generated.put(buildGetAutomationSchemaTool(projectId)); } + generated.put(buildBuildAutomationTool(projectId)); MCPUtility.stampGenerator(generated, AUTOMATION_MCP_GENERATOR_ID); String assetsFolder = AssetUtility.getProjectAssetsFolder(projectId); - String outputFileLoc = assetsFolder + "/mcp/pixel_mcp.json"; + String outputFileLoc = Paths.get(assetsFolder, "mcp", "pixel_mcp.json").toString(); JSONArray merged = MCPUtility.mergeGeneratedTools( MCPUtility.readMcpJson(outputFileLoc), generated, AUTOMATION_MCP_GENERATOR_ID, true); @@ -149,7 +159,7 @@ private static JSONObject buildTriggerAutomationTool(String projectId, String au + "per-workflow summary once complete (e.g. \"Indexed 20 files\")."; } if (hasDbNodes) { - description += " This automation has database nodes that accept SQL queries — call GetAutomationSchema first" + description += " This automation has database nodes that accept SQL queries - call GetAutomationSchema first" + " to discover the exact table and column names before writing SQL."; } tool.put("description", description); @@ -252,6 +262,54 @@ private static boolean hasPlaygroundDbNodes(String automationJson) { return false; } + /** + * Builds the {@code BuildAutomation} MCP tool - lets an agent in Playground generate or edit + * this project's automation from a plain-English description. Execution mode is ASK so the user + * can review the generated document before it is saved. + */ + private static JSONObject buildBuildAutomationTool(String projectId) { + JSONObject tool = new JSONObject(); + tool.put("name", "BuildAutomation"); + tool.put("title", "Build / Edit Automation"); + tool.put("description", + "Generates or edits this project's automation from a plain-English description. " + + "The model iteratively gathers context (database schema, available reactors) before producing a complete automation document. " + + "Does NOT save automatically - call SaveAutomation with the returned JSON to persist. " + + "Use currentDoc to pass the existing automation JSON (base64-encoded) for edit mode."); + + JSONObject projectProp = new JSONObject(); + projectProp.put("type", "string"); + projectProp.put("description", "The project ID for this automation. Always use: " + projectId); + projectProp.put("default", projectId); + + JSONObject descProp = new JSONObject(); + descProp.put("type", "string"); + descProp.put("description", "Plain-English description of what the automation should do, or how to modify the existing one. Will be base64-encoded automatically if needed."); + + JSONObject currentDocProp = new JSONObject(); + currentDocProp.put("type", "string"); + currentDocProp.put("description", "Optional base64-encoded JSON of the current automation document. Include this to edit rather than generate from scratch."); + + JSONObject properties = new JSONObject(); + properties.put(ReactorKeysEnum.PROJECT.getKey(), projectProp); + properties.put(AutomationConstants.DOC_DESCRIPTION, descProp); + properties.put("currentDoc", currentDocProp); + + JSONObject inputSchema = new JSONObject(); + inputSchema.put("type", "object"); + inputSchema.put("title", "BuildAutomation_Arguments"); + inputSchema.put("properties", properties); + inputSchema.put("required", new JSONArray().put(ReactorKeysEnum.PROJECT.getKey()).put(AutomationConstants.DOC_DESCRIPTION)); + tool.put("inputSchema", inputSchema); + + JSONObject meta = new JSONObject(); + meta.put(MCPUtility.SMSS_FUNCTION_NAME, "BuildAutomation"); + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPExecution.ASK.getValue()); + tool.put("_meta", meta); + + return tool; + } + /** Builds the auto-executable {@code GetAutomationSchema} companion tool. */ private static JSONObject buildGetAutomationSchemaTool(String projectId) { JSONObject tool = new JSONObject(); @@ -259,7 +317,7 @@ private static JSONObject buildGetAutomationSchemaTool(String projectId) { tool.put("title", "Get Automation Database Schema"); tool.put("description", "Returns the physical table and column names for database nodes in this automation that accept SQL input. " - + "Call this before TriggerAutomation when you need to write a SQL query — it gives you the exact " + + "Call this before TriggerAutomation when you need to write a SQL query - it gives you the exact " + "table and column names available in the database."); JSONObject projectProp = new JSONObject(); diff --git a/src/prerna/reactor/automation/AutomationRunEngine.java b/src/prerna/reactor/automation/AutomationRunEngine.java index 848fb0c957a..ee532022440 100644 --- a/src/prerna/reactor/automation/AutomationRunEngine.java +++ b/src/prerna/reactor/automation/AutomationRunEngine.java @@ -270,7 +270,7 @@ private static Map executeSingleNode(String runId, String projec private static ScheduledExecutorService startHeartbeat(String runId) { ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> { - Thread t = new Thread(r, "automation-heartbeat-" + runId.substring(0, 8)); + Thread t = new Thread(r, "automation-heartbeat-" + runId.substring(0, Math.min(8, runId.length()))); t.setDaemon(true); return t; }); diff --git a/src/prerna/reactor/automation/BuildAutomationReactor.java b/src/prerna/reactor/automation/BuildAutomationReactor.java new file mode 100644 index 00000000000..83f638ba398 --- /dev/null +++ b/src/prerna/reactor/automation/BuildAutomationReactor.java @@ -0,0 +1,259 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.engine.api.IModelEngine; +import prerna.engine.impl.model.RoomUtils; +import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.run.AgentRuntimeManager; +import prerna.reactor.agent.run.RunAgentRequest; +import prerna.reactor.agent.run.RunAgentResult; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Agentic automation builder. Uses the platform RunAgent harness to generate or + * modify an automation graph JSON document from a plain-English description. + * + *

All user-accessible engines are registered as MCP tools on the Room so the + * model can query database schema, inspect engine capabilities, etc. before + * producing the final document. The model's last response must be the raw + * automation JSON document. + * + *

Pixel: {@code BuildAutomation(project=["appId"], description=["base64desc"], engine=["modelId"], currentDoc=["base64json"])} + * + *

Returns the generated document JSON as a string. Does NOT persist - caller saves via SaveAutomation. + */ +public class BuildAutomationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(BuildAutomationReactor.class); + + private static final int DESCRIPTION_MAX_CHARS = 2000; + private static final int MAX_TURNS = 10; + private static final long BUILD_TIMEOUT_MS = 180_000L; + private static final String HARNESS_TYPE = "semoss"; + + private static final String CURRENT_DOC_KEY = "currentDoc"; + + private static final String SYSTEM_PROMPT = + "You are a workflow automation builder. Your job is to create (or modify) an automation graph JSON document " + + "from a plain-English description.\n\n" + + "Use the available tools to gather context before building (for example, query a database engine to " + + "understand its schema before writing SQL). When you are done, your FINAL response must be ONLY the raw " + + "automation JSON document - no prose, no code fences, no explanation. Just: {\"version\":1,...}\n\n" + + "## Document format\n\n" + + "The document must be:\n" + + "{\"version\":1,\"description\":\"\",\"graph\":{\"nodes\":[...],\"edges\":[]}}\n\n" + + "Each node: {\"id\":\"-\",\"type\":\"\",\"label\":\"\"," + + "\"position\":{\"x\":0,\"y\":0},\"outputVar\":\"\",\"config\":{...}}\n\n" + + "Node types and config templates:\n" + + " trigger {\"mode\":\"manual\"} outputVar: trigger_out\n" + + " database-engine {\"engineId\":\"\",\"operation\":\"query\",\"expression\":\"\",\"nlPrompt\":\"\",\"limit\":50,\"commit\":false} outputVar: db_out\n" + + " model-engine {\"engineId\":\"\",\"operation\":\"llm\",\"command\":\"\",\"context\":\"\",\"paramValues\":\"\",\"values\":\"\",\"image\":\"\",\"prompt\":\"\",\"entities\":\"\"} outputVar: model_out\n" + + " vector-engine {\"engineId\":\"\",\"operation\":\"search\",\"command\":\"\",\"limit\":5,\"filters\":\"\",\"metaFilters\":\"\",\"filePath\":\"\",\"source\":\"\",\"space\":\"\",\"filePaths\":\"\",\"paramValues\":\"\",\"fileNames\":\"\"} outputVar: vector_out\n" + + " storage-engine {\"engineId\":\"\",\"operation\":\"list\",\"storagePath\":\"/\",\"filePath\":\"\",\"metadata\":\"\"} outputVar: storage_out\n" + + " function-engine {\"engineId\":\"\",\"operation\":\"execute\",\"params\":\"\"} outputVar: fn_out\n" + + " app {\"pixel\":\"\",\"appId\":\"\"} outputVar: pixel_out\n" + + " wait {\"seconds\":\"5\"} outputVar: wait_out\n\n" + + "## Building rules\n" + + "1. First node MUST be trigger: id=\"trigger-1\", type=\"trigger\".\n" + + "2. Use only node types from the list above.\n" + + "3. Set engineId from the available engines listed in the user message. Use \"\" if no match.\n" + + "4. outputVar must be unique across all nodes.\n" + + "5. Label: action-oriented verb phrase (e.g. \"Search claims database\", \"Draft summary email\").\n" + + "6. Variable substitution: reference upstream outputVars with ${varName}. In SQL wrap in single quotes: WHERE col = '${db_out}'.\n" + + "7. NEVER use SQL parameterized syntax ($1, ?, :param) - unsupported.\n" + + "8. model-engine \"command\": plain instruction only. Set \"context\" to the upstream var (e.g. \"${db_out}\") - the model receives real data at runtime.\n" + + "9. For database-engine nodes, call the get_schema tool (passing the engineId as database_id) to look up the exact table and column names before writing SQL. Never guess column names.\n" + + "10. For app nodes, use the reactor/pixel information in the user message to write valid pixel expressions.\n" + + "11. Edit mode: when an existing document is provided, preserve nodes not affected by the user's request.\n" + + "12. Format SQL with newlines and indentation: keywords (SELECT, FROM, WHERE, JOIN, ORDER BY, LIMIT, etc.) each on their own line, with clause contents indented.\n" + + "13. For every database-engine node, always include \"nlPrompt\" - a one-sentence plain-English description of what data the query fetches.\n"; + + public BuildAutomationReactor() { + this.keysToGet = new String[] { + ReactorKeysEnum.PROJECT.getKey(), + AutomationConstants.DOC_DESCRIPTION, + ReactorKeysEnum.ENGINE.getKey(), + CURRENT_DOC_KEY + }; + this.keyRequired = new int[] { 1, 1, 0, 0 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } + + String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); + String description = this.keyValue.get(AutomationConstants.DOC_DESCRIPTION); + String engineId = this.keyValue.get(ReactorKeysEnum.ENGINE.getKey()); + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access."); + } + + if (description == null || description.trim().isEmpty()) { + throw new IllegalArgumentException("A description of what the automation should do is required."); + } + try { + description = new String( + Base64.getDecoder().decode(description.trim()), + StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + // Not base64-encoded - use as-is + } + if (description.length() > DESCRIPTION_MAX_CHARS) { + description = description.substring(0, DESCRIPTION_MAX_CHARS); + } + + if (engineId == null || engineId.trim().isEmpty()) { + engineId = AutomationExecutionUtils.findFirstModelEngine(user); + } + if (engineId == null || engineId.trim().isEmpty()) { + throw new IllegalArgumentException( + "No AI model engine is available. Add a model engine connection to build an automation."); + } + if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { + throw new IllegalArgumentException( + "Model engine " + engineId + " does not exist or user does not have access."); + } + IModelEngine modelEngine = Utility.getModel(engineId); + if (modelEngine == null) { + throw new IllegalArgumentException( + "Model engine " + engineId + " could not be loaded. It may no longer exist."); + } + + String currentDocRaw = this.keyValue.get(CURRENT_DOC_KEY); + String currentDoc = null; + if (currentDocRaw != null && !currentDocRaw.trim().isEmpty()) { + try { + currentDoc = new String( + Base64.getDecoder().decode(currentDocRaw.trim()), + StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + currentDoc = currentDocRaw; + } + if (currentDoc != null && currentDoc.length() > 50_000) { + currentDoc = currentDoc.substring(0, 50_000); + } + } + + StringBuilder initialMsg = new StringBuilder(); + initialMsg.append(AutomationExecutionUtils.buildAvailableEnginesSection(user)).append("\n"); + if (currentDoc != null) { + classLogger.info("BuildAutomation edit mode: project={}, docLength={}", projectId, currentDoc.length()); + initialMsg.append("## Existing automation to modify\n").append(currentDoc).append("\n\n"); + initialMsg.append("## User modification request\n").append(description.trim()); + } else { + initialMsg.append("## User request\n").append(description.trim()); + } + + // Fresh room per build request - isolated tool-call context, no history bleed + String pidClean = projectId.replace("-", ""); + String roomId = "automationbuild" + + pidClean.substring(0, Math.min(8, pidClean.length())) + + Long.toString(System.currentTimeMillis(), 36); + + Map options = AutomationExecutionUtils.buildEngineMcpOptions(user, SYSTEM_PROMPT); + + RoomUtils.createRoomIfNotExists(roomId, this.insight, modelEngine, initialMsg.toString(), + null, options, null, projectId, null); + + RunAgentRequest request = new RunAgentRequest( + roomId, initialMsg.toString(), engineId, HARNESS_TYPE, null, + MAX_TURNS, 0, null, null, null, null, this.insight); + + classLogger.info("BuildAutomation starting RunAgent: project={} roomId={}", projectId, roomId); + + RunAgentResult handle = AgentRuntimeManager.get().run(request); + Map result; + try { + result = AgentRuntimeManager.get().waitForRun(handle.getRunId(), this.insight, BUILD_TIMEOUT_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Build interrupted.", e); + } + + String status = (String) result.get("status"); + if ("FAILED".equals(status)) { + String errMsg = (String) result.get("errorMessage"); + classLogger.error("BuildAutomation run failed: project={} error={}", projectId, errMsg); + throw new RuntimeException("AI generation failed: " + (errMsg != null ? errMsg : "unknown error")); + } + + String finalText = (String) result.get("finalText"); + if (finalText == null || finalText.isBlank()) { + throw new IllegalStateException( + "The AI model did not return a response. Try again or start with a blank automation."); + } + + String docJson = AutomationExecutionUtils.stripCodeFences(finalText.trim()); + AutomationExecutionUtils.validateGeneratedDoc(docJson); + + classLogger.info("BuildAutomation finished: project={}", projectId); + return new NounMetadata(docJson, PixelDataType.CONST_STRING, PixelOperationType.OPERATION); + } + + @Override + public String getReactorDescription() { + return "Agentic automation builder using the platform RunAgent harness. Generates or edits an automation " + + "graph document from a plain-English description. All user engines are registered as MCP tools " + + "so the model can query schema and capabilities before building. Does NOT persist - " + + "caller saves via SaveAutomation."; + } + + @Override + protected String getDescriptionForKey(String key) { + return switch (key) { + case "project" -> "Project ID that will own this automation."; + case "description" -> "Base64-encoded plain-English description of what the automation should do."; + case "engine" -> "Optional model engine ID to use. Defaults to the first available MODEL engine."; + case CURRENT_DOC_KEY -> "Optional base64-encoded JSON of an existing automation document. When provided, the model modifies the existing document rather than generating from scratch."; + default -> super.getDescriptionForKey(key); + }; + } +} diff --git a/src/prerna/reactor/automation/ExplainAutomationReactor.java b/src/prerna/reactor/automation/ExplainAutomationReactor.java new file mode 100644 index 00000000000..2eea7fa4d55 --- /dev/null +++ b/src/prerna/reactor/automation/ExplainAutomationReactor.java @@ -0,0 +1,223 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.engine.api.IModelEngine; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.AssetUtility; +import prerna.util.Utility; + +/** + * Generates a plain-English explanation of what an automation does when run. + * Reads the saved automation.json, sends it to a model engine with a narration prompt, + * and returns the explanation as a string. + * + *

Suitable for sharing with non-technical stakeholders or onboarding new team members. + * + *

Uses the same model-engine resolution logic as {@link GenerateAutomationReactor}: + * uses the {@code engine} param if provided, otherwise falls back to the first accessible MODEL engine. + * + *

Pixel: {@code ExplainAutomation(project=["appId"])} + */ +public class ExplainAutomationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(ExplainAutomationReactor.class); + + /** + * Narration prompt for the full explain mode (saved doc, 2-3 sentences for stakeholders). + * Used when no {@code content} parameter is provided. + */ + private static final String NARRATION_SYSTEM_PROMPT = + "You are a helpful assistant that explains software automation workflows to non-technical users. " + + "Given the following automation JSON definition, write a 2-3 sentence plain-English explanation " + + "of what this automation does when it runs. " + + "Write for a non-technical audience — do not mention JSON, nodes, config keys, or technical terms. " + + "Start your response with: 'When you run this automation, it will...'"; + + /** + * Suggest prompt for the description field (in-memory doc, 1 sentence for the description field). + * Used when a {@code content} parameter is provided (current unsaved state from the FE). + */ + private static final String SUGGEST_SYSTEM_PROMPT = + "You generate one-sentence descriptions for workflow automations. " + + "Given an automation graph JSON, write a single clear sentence (under 20 words) describing what the automation does. " + + "Be specific about the actions taken, not about node types or technical structure. " + + "Start with an active verb. No quotes, no period at the end, no explanation. " + + "Examples: 'Queries open claims and drafts a daily summary email for case managers', " + + "'Searches the knowledge base for relevant documents and generates a response using AI', " + + "'Uploads processed files to cloud storage and updates the claims database'."; + + private static final String CONTENT_KEY = "content"; + + public ExplainAutomationReactor() { + this.keysToGet = new String[] { + ReactorKeysEnum.PROJECT.getKey(), + ReactorKeysEnum.ENGINE.getKey(), + CONTENT_KEY, + }; + this.keyRequired = new int[] { 1, 0, 0 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } + + String projectId = this.keyValue.get(this.keysToGet[0]); + String engineId = this.keyValue.get(this.keysToGet[1]); + String contentEncoded = this.keyValue.get(CONTENT_KEY); + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { + throw new IllegalArgumentException( + "Project does not exist or user does not have access."); + } + + // Determine mode: suggest (in-memory content provided) vs. narrate (read saved file) + boolean suggestMode = contentEncoded != null && !contentEncoded.trim().isEmpty(); + String doc; + String systemPrompt; + if (suggestMode) { + try { + doc = new String( + java.util.Base64.getDecoder().decode(contentEncoded.trim()), + StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + doc = contentEncoded; // not base64-encoded, use as-is + } + if (doc.length() > 50_000) { + doc = doc.substring(0, 50_000); + } + systemPrompt = SUGGEST_SYSTEM_PROMPT; + classLogger.info("ExplainAutomationReactor: suggest mode (in-memory content), project={}", projectId); + } else { + classLogger.info("ExplainAutomationReactor: narrate mode (saved file), project={}", projectId); + doc = loadCurrentDoc(projectId); + if (doc.length() > 50_000) { + doc = doc.substring(0, 50_000); + } + systemPrompt = NARRATION_SYSTEM_PROMPT; + } + + // Resolve model engine — provided ID or first accessible MODEL engine + if (engineId == null || engineId.trim().isEmpty()) { + engineId = AutomationExecutionUtils.findFirstModelEngine(user); + } + if (engineId == null || engineId.trim().isEmpty()) { + throw new IllegalArgumentException( + "No AI model engine is available. Add a model engine connection to use this feature."); + } + if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { + throw new IllegalArgumentException( + "Model engine " + engineId + " does not exist or user does not have access."); + } + + IModelEngine modelEngine = Utility.getModel(engineId); + if (modelEngine == null) { + throw new IllegalArgumentException( + "Model engine " + engineId + " could not be loaded. It may no longer exist."); + } + + classLogger.info("ExplainAutomationReactor: calling model engine {} for project {}", engineId, projectId); + + Map paramMap = new HashMap<>(); + paramMap.put("use_history", false); + + Map response; + try { + String userMessage = "Automation definition:\n" + doc; + response = modelEngine.ask(systemPrompt + "\n\n" + userMessage, + null, this.insight, paramMap).toMap(); + } catch (Exception e) { + classLogger.error("LLM call failed for ExplainAutomation on project {}", projectId, e); + throw new RuntimeException("AI explanation failed: " + e.getMessage(), e); + } + + String explanation = AutomationExecutionUtils.extractResponseText(response); + if (explanation == null || explanation.isBlank()) { + throw new IllegalStateException( + "The AI model did not return an explanation. Try again."); + } + + classLogger.info("ExplainAutomationReactor: completed for project {}", projectId); + return new NounMetadata(explanation.trim(), PixelDataType.CONST_STRING, PixelOperationType.OPERATION); + } + + private String loadCurrentDoc(String projectId) { + try { + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File automationFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); + if (automationFile.exists() && automationFile.isFile()) { + return Files.readString(automationFile.toPath(), StandardCharsets.UTF_8); + } + } catch (IOException e) { + classLogger.warn("Could not read automation.json for project {} — returning empty doc", projectId, e); + } + return "{\"version\":1,\"graph\":{\"nodes\":[],\"edges\":[]}}"; + } + + @Override + public String getReactorDescription() { + return "Generates a plain-English explanation of what a saved automation does when run, " + + "suitable for sharing with non-technical stakeholders. " + + "Reads the saved automation.json and narrates it using a model engine."; + } + + @Override + protected String getDescriptionForKey(String key) { + return switch (key) { + case "project" -> "The project ID of the automation to explain."; + case "engine" -> "Optional model engine ID to use. Defaults to the first accessible MODEL engine."; + case CONTENT_KEY -> "Optional base64-encoded JSON of the current automation (from FE in-memory state). " + + "When provided, produces a one-sentence description suitable for the description field " + + "instead of reading the saved file and narrating 2-3 sentences."; + default -> super.getDescriptionForKey(key); + }; + } +} diff --git a/src/prerna/reactor/automation/GenerateAutomationReactor.java b/src/prerna/reactor/automation/GenerateAutomationReactor.java index 72bf02fdaaf..3a758acc9be 100644 --- a/src/prerna/reactor/automation/GenerateAutomationReactor.java +++ b/src/prerna/reactor/automation/GenerateAutomationReactor.java @@ -28,7 +28,6 @@ package prerna.reactor.automation; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -41,10 +40,7 @@ import prerna.auth.User; import prerna.auth.utils.SecurityEngineUtils; import prerna.auth.utils.SecurityProjectUtils; -import prerna.engine.api.IDatabaseEngine; import prerna.engine.api.IModelEngine; -import prerna.engine.api.IRDBMSEngine; -import prerna.project.api.IProject; import prerna.reactor.AbstractReactor; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; @@ -65,17 +61,13 @@ * expected to display it for review and save it via {@code SaveAutomation} — this reactor does NOT * persist anything. */ -public final class GenerateAutomationReactor extends AbstractReactor { +public class GenerateAutomationReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(GenerateAutomationReactor.class); private static final int DESCRIPTION_MAX_CHARS = 1000; private static final String PENDING_SQL = "PENDING_SQL_GENERATION"; - /** Engine types used to fetch context for the prompt. */ - private static final List SUPPORTED_ENGINE_TYPES = Arrays.asList( - "DATABASE", "MODEL", "VECTOR", "STORAGE", "FUNCTION"); - /** * Pass 1 system prompt: structural generation only. * The LLM picks node types, engines, labels, and output vars. @@ -234,7 +226,7 @@ public NounMetadata execute() { // Resolve model engine — use provided ID or fall back to first available MODEL engine if (engineId == null || engineId.trim().isEmpty()) { - engineId = findFirstModelEngine(user); + engineId = AutomationExecutionUtils.findFirstModelEngine(user); } if (engineId == null || engineId.trim().isEmpty()) { throw new IllegalArgumentException( @@ -272,7 +264,7 @@ public NounMetadata execute() { paramMap.put("use_history", false); // -- Pass 1: structural generation (or edit if current doc provided) ------------- - String enginesSection = buildAvailableEnginesSection(user); + String enginesSection = AutomationExecutionUtils.buildAvailableEnginesSection(user); String pass1Message; String systemPrompt1; if (currentDoc != null) { @@ -286,8 +278,8 @@ public NounMetadata execute() { pass1Message = enginesSection + "\n## User's request\n" + description.trim(); } String raw = callLlm(modelEngine, systemPrompt1, pass1Message, paramMap, projectId); - raw = stripCodeFences(raw); - validateGeneratedDoc(raw); + raw = AutomationExecutionUtils.stripCodeFences(raw); + AutomationExecutionUtils.validateGeneratedDoc(raw); // -- Pass 2: schema-aware refinement (when DB nodes or app nodes are present) ---- List dbEngineIds = extractDatabaseEngineIds(raw); @@ -297,15 +289,15 @@ public NounMetadata execute() { dbEngineIds.size(), hasAppNodes, projectId); StringBuilder pass2Message = new StringBuilder("## Workflow document from pass 1\n").append(raw).append("\n"); if (!dbEngineIds.isEmpty()) { - pass2Message.append("\n").append(buildSchemaForEngineIds(dbEngineIds)); + pass2Message.append("\n").append(AutomationExecutionUtils.buildSchemaForEngineIds(dbEngineIds)); } if (hasAppNodes) { - pass2Message.append("\n").append(buildReactorListSection(projectId)); + pass2Message.append("\n").append(AutomationExecutionUtils.buildReactorListSection(projectId)); } try { String refined = callLlm(modelEngine, SYSTEM_PROMPT_REFINE, pass2Message.toString(), paramMap, projectId); - refined = stripCodeFences(refined); - validateGeneratedDoc(refined); + refined = AutomationExecutionUtils.stripCodeFences(refined); + AutomationExecutionUtils.validateGeneratedDoc(refined); raw = refined; } catch (Exception e) { classLogger.warn("GenerateAutomation pass 2 failed — returning pass 1 result. Reason: {}", e.getMessage()); @@ -318,7 +310,6 @@ public NounMetadata execute() { // -- Private helpers ------------------------------------------------------------- - /** Calls the model and returns the response text, throwing on failure. */ private String callLlm(IModelEngine modelEngine, String systemPrompt, String userMessage, Map paramMap, String projectId) { Map response; @@ -328,7 +319,7 @@ private String callLlm(IModelEngine modelEngine, String systemPrompt, String use classLogger.error("LLM call failed for GenerateAutomation on project {}", projectId, e); throw new RuntimeException("AI generation failed: " + e.getMessage(), e); } - String text = extractResponseText(response); + String text = AutomationExecutionUtils.extractResponseText(response); if (text == null || text.isBlank()) { throw new IllegalStateException( "The AI model did not return a response. Try again or start with a blank automation."); @@ -394,188 +385,6 @@ private static boolean extractHasAppNodes(String docJson) { return false; } - /** - * Returns the list of custom reactors available in the given project, formatted as a - * bulleted list for the pass 2 prompt. Uses the same source as the FE reactor browser. - */ - private static String buildReactorListSection(String projectId) { - StringBuilder sb = new StringBuilder("## Available custom reactors in this project\n"); - try { - IProject project = Utility.getProject(projectId); - if (project == null) { - sb.append("(project not found — leave app node pixel as-is)\n"); - return sb.toString(); - } - java.util.TreeSet reactors = project.getAvailableReactors(); - if (reactors == null || reactors.isEmpty()) { - sb.append("(none — leave app node pixel as a comment placeholder)\n"); - return sb.toString(); - } - for (String name : reactors) { - sb.append("- ").append(name).append("\n"); - } - } catch (Exception e) { - classLogger.warn("Failed to build reactor list for project {}", projectId, e); - sb.append("(reactor list unavailable — leave app node pixel as a comment placeholder)\n"); - } - return sb.toString(); - } - - /** - * Fetches the table/column schema (with data types) for the given engine IDs. - * Uses the same metamodel API as {@code TextToSQLReactor}. Data types prevent the LLM - * from generating type-mismatched WHERE clauses (e.g. comparing VARCHAR to an integer). - */ - private static String buildSchemaForEngineIds(List engineIds) { - StringBuilder sb = new StringBuilder("## Database schema\n"); - for (String engineId : engineIds) { - try { - IDatabaseEngine dbEngine = Utility.getDatabase(engineId); - if (!(dbEngine instanceof IRDBMSEngine rdbms)) { - sb.append("Database id=\"").append(engineId).append("\": (non-relational — no table schema)\n"); - continue; - } - sb.append("Database id=\"").append(engineId).append("\":\n"); - List tables = rdbms.getPixelConcepts(); - if (tables == null || tables.isEmpty()) { - sb.append(" (no tables found)\n"); - continue; - } - for (String table : tables) { - List columns = rdbms.getPixelSelectors(table); - sb.append(" - ").append(table).append(" ("); - if (columns != null && !columns.isEmpty()) { - StringBuilder cols = new StringBuilder(); - for (String col : columns) { - int sep = col.lastIndexOf("__"); - String colName = sep >= 0 ? col.substring(sep + 2) : col; - String dataType = null; - try { - String physicalUri = rdbms.getPhysicalUriFromPixelSelector(col); - if (physicalUri != null) { - dataType = rdbms.getDataTypes(physicalUri); - } - } catch (Exception ignored) { - // Data type fetch is best-effort; column name alone is still useful - } - if (cols.length() > 0) cols.append(", "); - cols.append(colName); - if (dataType != null && !dataType.isBlank()) { - cols.append(": ").append(dataType.trim()); - } - } - sb.append(cols); - } - sb.append(")\n"); - } - } catch (Exception e) { - classLogger.warn("Failed to fetch schema for engine {}", engineId, e); - sb.append("Database id=\"").append(engineId).append("\": (schema unavailable)\n"); - } - } - return sb.toString(); - } - - /** - * Returns the ID of the first MODEL-type engine the user has access to, or null if none. - */ - private static String findFirstModelEngine(User user) { - try { - List> engines = SecurityEngineUtils.getUserEngineList( - user, List.of("MODEL"), null, false, null, null, null, "1", "0", null); - if (engines != null && !engines.isEmpty()) { - Object id = engines.get(0).get("database_id"); - return id != null ? String.valueOf(id) : null; - } - } catch (Exception e) { - classLogger.warn("Failed to auto-discover model engine", e); - } - return null; - } - - /** - * Builds the "## Available engines" section appended to the pass 1 user message. - * Lists each engine the user has access to so the LLM can populate engineId fields. - */ - private static String buildAvailableEnginesSection(User user) { - StringBuilder sb = new StringBuilder("## Available engines\n"); - try { - List> engines = SecurityEngineUtils.getUserEngineList( - user, SUPPORTED_ENGINE_TYPES, null, false, null, null, null, "50", "0", null); - if (engines == null || engines.isEmpty()) { - sb.append("None available — leave engineId as empty string.\n"); - return sb.toString(); - } - for (Map engine : engines) { - String id = engine.getOrDefault("database_id", "").toString(); - String name = engine.getOrDefault("database_name", "").toString(); - String type = engine.getOrDefault("engine_type", "").toString().toUpperCase(); - sb.append("- type=").append(type) - .append(" id=\"").append(id) - .append("\" name=\"").append(name).append("\"\n"); - } - } catch (Exception e) { - classLogger.warn("Failed to build engine list for generation prompt", e); - sb.append("(engine list unavailable — leave engineId as empty string)\n"); - } - return sb.toString(); - } - - /** - * Extracts the text content from the model's response map. - * Handles both {@code response} string and {@code output}/{@code content} keys. - */ - private static String extractResponseText(Map response) { - if (response == null) return null; - Object resp = response.get("response"); - if (resp instanceof String s && !s.isBlank()) return s; - Object output = response.get("output"); - if (output instanceof String s && !s.isBlank()) return s; - Object content = response.get("content"); - if (content instanceof String s && !s.isBlank()) return s; - return null; - } - - /** Strips leading/trailing markdown code fences (```json ... ``` or ``` ... ```). */ - private static String stripCodeFences(String raw) { - String trimmed = raw.strip(); - if (trimmed.startsWith("```")) { - int firstNewline = trimmed.indexOf('\n'); - if (firstNewline != -1) { - trimmed = trimmed.substring(firstNewline + 1); - } - if (trimmed.endsWith("```")) { - trimmed = trimmed.substring(0, trimmed.lastIndexOf("```")).strip(); - } - } - return trimmed; - } - - /** - * Throws if the generated doc is not parseable JSON or missing required structure. - * Does not enforce strict schema — just enough to avoid crashing the FE. - */ - private static void validateGeneratedDoc(String raw) { - try { - JSONObject doc = new JSONObject(raw); - if (!doc.has("graph")) { - throw new IllegalStateException("Generated document is missing the 'graph' field."); - } - JSONObject graph = doc.getJSONObject("graph"); - if (!graph.has("nodes")) { - throw new IllegalStateException("Generated graph is missing the 'nodes' array."); - } - JSONArray nodes = graph.getJSONArray("nodes"); - if (nodes.length() == 0) { - throw new IllegalStateException("Generated graph has no nodes."); - } - } catch (org.json.JSONException e) { - classLogger.warn("Generated automation doc is not valid JSON: {}", raw, e); - throw new IllegalStateException( - "The AI model returned an invalid response. Please try again with a different description.", e); - } - } - @Override public String getReactorDescription() { return "Uses an AI model to scaffold a starter automation graph from a plain-English description. " diff --git a/src/prerna/reactor/automation/GenerateNodeLabelReactor.java b/src/prerna/reactor/automation/GenerateNodeLabelReactor.java new file mode 100644 index 00000000000..101b17cb36e --- /dev/null +++ b/src/prerna/reactor/automation/GenerateNodeLabelReactor.java @@ -0,0 +1,244 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.engine.api.IModelEngine; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Generates a short, action-oriented label (2–4 words) for an automation node + * based on its type and configuration. Uses the first available MODEL engine. + * + *

Pixel: {@code GenerateNodeLabel(project=["appId"], type=["model-engine"], config=["base64json"])} + */ +public class GenerateNodeLabelReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GenerateNodeLabelReactor.class); + + private static final String KEY_TYPE = "type"; + private static final String KEY_CONFIG = "config"; + + private static final String SYSTEM_PROMPT = + "You generate short, action-oriented labels for workflow automation steps. " + + "Given the step type and key configuration details, respond with ONLY the label - " + + "2 to 4 words maximum, plain English, no quotes, no punctuation at the end, no explanation. " + + "The label should describe what the step DOES, not what type of node it is. " + + "Examples: 'Search claims data', 'Draft email reply', 'Fetch open tickets', 'Summarize results', " + + "'Upload report file', 'Query veteran records', 'Pause 30 seconds'."; + + public GenerateNodeLabelReactor() { + this.keysToGet = new String[] { + ReactorKeysEnum.PROJECT.getKey(), + KEY_TYPE, + KEY_CONFIG, + }; + this.keyRequired = new int[] { 1, 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } + + String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); + String nodeType = this.keyValue.get(KEY_TYPE); + String configEncoded = this.keyValue.get(KEY_CONFIG); + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access."); + } + + // Decode base64 config JSON sent by FE + String configJson; + try { + configJson = new String(Base64.getDecoder().decode(configEncoded.trim()), java.nio.charset.StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + configJson = configEncoded; + } + + String engineId = AutomationExecutionUtils.findFirstModelEngine(user); + if (engineId == null || engineId.isBlank()) { + throw new IllegalArgumentException( + "No AI model engine is available. Add a model engine connection to use this feature."); + } + if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { + throw new IllegalArgumentException( + "Model engine " + engineId + " does not exist or user does not have access."); + } + + IModelEngine modelEngine = Utility.getModel(engineId); + if (modelEngine == null) { + throw new IllegalArgumentException( + "Model engine " + engineId + " could not be loaded."); + } + + String userMessage = buildUserMessage(nodeType, configJson); + classLogger.info("GenerateNodeLabel: project={}, type={}", projectId, nodeType); + + Map paramMap = new HashMap<>(); + paramMap.put("use_history", false); + + Map response; + try { + response = modelEngine.ask(SYSTEM_PROMPT + "\n\n" + userMessage, + null, this.insight, paramMap).toMap(); + } catch (Exception e) { + classLogger.error("LLM call failed for GenerateNodeLabel on project {}", projectId, e); + throw new RuntimeException("Label generation failed: " + e.getMessage(), e); + } + + String label = AutomationExecutionUtils.extractResponseText(response); + if (label == null || label.isBlank()) { + throw new IllegalStateException("The AI model did not return a label. Try again."); + } + + // Strip surrounding quotes the model sometimes adds, and cap length + label = label.strip().replaceAll("^[\"']+|[\"']+$", ""); + if (label.length() > 50) { + label = label.substring(0, 50); + } + + return new NounMetadata(label, PixelDataType.CONST_STRING, PixelOperationType.OPERATION); + } + + /** + * Builds a human-readable summary of the node config for the LLM prompt. + * Skips engine IDs and other technical fields the model doesn't need to understand. + */ + @SuppressWarnings("unchecked") + private static String buildUserMessage(String nodeType, String configJson) { + StringBuilder sb = new StringBuilder(); + sb.append("Step type: ").append(humanNodeType(nodeType)).append("\n"); + + try { + Map cfg = AutomationExecutionUtils.GSON.fromJson(configJson, AutomationExecutionUtils.MAP_TYPE); + if (cfg == null) { + sb.append("Config: (none)"); + return sb.toString(); + } + + switch (nodeType) { + case "model-engine" -> { + appendField(sb, "Instruction", cfg.get("command")); + appendField(sb, "Input data", cfg.get("context")); + appendField(sb, "Operation", cfg.get("operation")); + } + case "database-engine" -> { + appendField(sb, "SQL query", cfg.get("expression")); + appendField(sb, "Operation", cfg.get("operation")); + } + case "vector-engine" -> { + appendField(sb, "Search query", cfg.get("command")); + appendField(sb, "Operation", cfg.get("operation")); + appendField(sb, "File", cfg.get("filePath")); + } + case "storage-engine" -> { + appendField(sb, "Operation", cfg.get("operation")); + appendField(sb, "Path", cfg.get("storagePath")); + appendField(sb, "File", cfg.get("filePath")); + } + case "function-engine" -> { + appendField(sb, "Parameters", cfg.get("params")); + appendField(sb, "Operation", cfg.get("operation")); + } + case "app" -> { + appendField(sb, "Pixel expression", cfg.get("pixel")); + } + case "wait" -> { + appendField(sb, "Seconds", cfg.get("seconds")); + } + default -> { + // No config details for trigger or unknown types + } + } + } catch (Exception e) { + sb.append("Config: (unparseable)"); + } + + return sb.toString(); + } + + private static void appendField(StringBuilder sb, String fieldName, Object value) { + if (value == null) return; + String s = value.toString().trim(); + if (s.isEmpty() || s.equals("PENDING_SQL_GENERATION") || s.equals("PENDING_PIXEL_EXPRESSION")) return; + // Truncate very long values so they don't dominate the prompt + if (s.length() > 200) s = s.substring(0, 200) + "..."; + sb.append(fieldName).append(": ").append(s).append("\n"); + } + + private static String humanNodeType(String type) { + return switch (type) { + case "model-engine" -> "Ask AI (model engine)"; + case "database-engine" -> "Query database"; + case "vector-engine" -> "Search documents (vector engine)"; + case "storage-engine" -> "File storage"; + case "function-engine" -> "Custom function"; + case "app" -> "Run pixel / app reactor"; + case "wait" -> "Pause / wait"; + case "trigger" -> "Trigger"; + default -> type; + }; + } + + @Override + public String getReactorDescription() { + return "Generates a short 2–4 word action-oriented label for an automation node based on its type and config. " + + "Uses the first available MODEL engine. Returns the suggested label as a string."; + } + + @Override + protected String getDescriptionForKey(String key) { + return switch (key) { + case "project" -> "The project ID the automation belongs to."; + case "type" -> "The node type (e.g. model-engine, database-engine, vector-engine)."; + case "config" -> "Base64-encoded JSON of the node's current config object."; + default -> super.getDescriptionForKey(key); + }; + } +} diff --git a/src/prerna/reactor/automation/GenerateRunSummaryReactor.java b/src/prerna/reactor/automation/GenerateRunSummaryReactor.java new file mode 100644 index 00000000000..acf88e96afd --- /dev/null +++ b/src/prerna/reactor/automation/GenerateRunSummaryReactor.java @@ -0,0 +1,159 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.engine.api.IModelEngine; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Generates a plain-English summary of a completed automation run using an LLM. + * Reads per-step output previews from the database and asks the model to describe + * what happened in 2-3 sentences suitable for a non-technical user. + * + *

Pixel: {@code GenerateRunSummary(project=["appId"], runId=["runId"])} + */ +public class GenerateRunSummaryReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GenerateRunSummaryReactor.class); + + private static final String KEY_RUN_ID = "runId"; + + private static final String SYSTEM_PROMPT = + "You summarize completed automation workflow runs for non-technical users. " + + "Given the run status and a list of steps with their outputs, write a 2-3 sentence " + + "plain-English summary of what happened: what the automation did, what it found or produced, " + + "and (if applicable) what went wrong. Be specific about data where available. " + + "Do not use technical jargon, node IDs, or implementation details. " + + "Do not start with 'The automation'. Respond with only the summary - no preamble, no headers."; + + public GenerateRunSummaryReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), KEY_RUN_ID }; + this.keyRequired = new int[] { 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } + + String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); + String runId = this.keyValue.get(KEY_RUN_ID); + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access."); + } + + Map runDetail = AutomationDatabaseUtility.getRunDetail(runId); + if (runDetail == null) { + throw new IllegalArgumentException("Run not found: " + runId); + } + + // Guard against summarizing an in-progress run - the result would be incomplete. + Object runStatus = runDetail.get(AutomationConstants.STATUS); + if (AutomationConstants.STATUS_RUNNING.equals(runStatus)) { + throw new IllegalArgumentException("Run " + runId + " is still in progress. Wait for it to complete."); + } + + List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(runId); + if (nodeOutputs.isEmpty()) { + classLogger.warn("GenerateRunSummary: no node outputs found for run {}", runId); + } + + String engineId = AutomationExecutionUtils.findFirstModelEngine(user); + if (engineId == null || engineId.isBlank()) { + throw new IllegalArgumentException( + "No AI model engine is available. Add a model engine connection to use this feature."); + } + if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { + throw new IllegalArgumentException( + "Model engine " + engineId + " does not exist or user does not have access."); + } + + IModelEngine modelEngine = Utility.getModel(engineId); + if (modelEngine == null) { + throw new IllegalArgumentException("Model engine " + engineId + " could not be loaded."); + } + + String userMessage = AutomationExecutionUtils.buildRunSummaryPrompt(runDetail, nodeOutputs); + + Map paramMap = new HashMap<>(); + paramMap.put("use_history", false); + + Map response; + try { + response = modelEngine.ask(SYSTEM_PROMPT + "\n\n" + userMessage, + null, this.insight, paramMap).toMap(); + } catch (Exception e) { + classLogger.error("LLM call failed for GenerateRunSummary on run {}", runId, e); + throw new RuntimeException("Summary generation failed: " + e.getMessage(), e); + } + + String summary = AutomationExecutionUtils.extractResponseText(response); + if (summary == null || summary.isBlank()) { + throw new IllegalStateException("The AI model did not return a summary. Try again."); + } + + classLogger.info("GenerateRunSummary completed: project={}, runId={}", projectId, runId); + return new NounMetadata(summary.strip(), PixelDataType.CONST_STRING, PixelOperationType.OPERATION); + } + + @Override + public String getReactorDescription() { + return "Generates a plain-English 2-3 sentence summary of a completed automation run " + + "using an LLM. Reads per-step results from the database. Returns the summary string."; + } + + @Override + protected String getDescriptionForKey(String key) { + return switch (key) { + case "project" -> "The project ID the automation belongs to."; + case "runId" -> "The run ID to summarize."; + default -> super.getDescriptionForKey(key); + }; + } +} diff --git a/src/prerna/reactor/automation/GenerateSQLReactor.java b/src/prerna/reactor/automation/GenerateSQLReactor.java deleted file mode 100644 index 09f5059852a..00000000000 --- a/src/prerna/reactor/automation/GenerateSQLReactor.java +++ /dev/null @@ -1,244 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.reactor.automation; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import prerna.auth.User; -import prerna.auth.utils.SecurityEngineUtils; -import prerna.engine.api.IDatabaseEngine; -import prerna.engine.api.IModelEngine; -import prerna.engine.api.IRDBMSEngine; -import prerna.reactor.AbstractReactor; -import prerna.sablecc2.om.PixelDataType; -import prerna.sablecc2.om.PixelOperationType; -import prerna.sablecc2.om.ReactorKeysEnum; -import prerna.sablecc2.om.nounmeta.NounMetadata; -import prerna.util.Utility; - -/** - * Uses an LLM to generate a SQL SELECT query for a given database engine from a plain-English description. - * - *

Pixel: {@code GenerateSQL(database=["engineId"], description=["base64EncodedDescription"])} - * - *

Fetches the engine's schema, finds the first available MODEL engine, and returns a SQL string. - * The caller is expected to review and edit the result before running it. - */ -public final class GenerateSQLReactor extends AbstractReactor { - - private static final Logger classLogger = LogManager.getLogger(GenerateSQLReactor.class); - - private static final String SYSTEM_PROMPT = """ -You are a SQL expert. Given a database schema and a plain-English description, write a single SQL SELECT query that retrieves the requested data. - -Rules: -- Use only the tables and columns that exist in the schema -- Use column types to write type-safe SQL — don't compare VARCHAR columns to integers -- Do not add conditions or filters not implied by the description -- Include a LIMIT clause appropriate to the request (e.g. LIMIT 100 for "get all X" queries) -- Return ONLY the SQL statement — no markdown, no explanation, no code fences - -## Database schema -"""; - - public GenerateSQLReactor() { - this.keysToGet = new String[] { - ReactorKeysEnum.DATABASE.getKey(), - AutomationConstants.DOC_DESCRIPTION, - ReactorKeysEnum.ENGINE.getKey() - }; - this.keyRequired = new int[] { 1, 1, 0 }; - } - - @Override - public NounMetadata execute() { - organizeKeys(); - - User user = this.insight.getUser(); - if (user == null) { - throw new IllegalArgumentException("You are not properly logged in."); - } - - String databaseId = this.keyValue.get(ReactorKeysEnum.DATABASE.getKey()); - String description = this.keyValue.get(AutomationConstants.DOC_DESCRIPTION); - String engineId = this.keyValue.get(ReactorKeysEnum.ENGINE.getKey()); - - if (databaseId == null || databaseId.trim().isEmpty()) { - throw new IllegalArgumentException("A database engine ID is required."); - } - if (!SecurityEngineUtils.userCanViewEngine(user, databaseId)) { - throw new IllegalArgumentException("Database engine does not exist or user does not have access."); - } - - if (description == null || description.trim().isEmpty()) { - throw new IllegalArgumentException("A description of the data to retrieve is required."); - } - // Decode base64-encoded description from FE - try { - description = new String( - java.util.Base64.getDecoder().decode(description.trim()), - java.nio.charset.StandardCharsets.UTF_8); - } catch (IllegalArgumentException e) { - // Not base64-encoded — use as-is - } - - // Resolve model engine - if (engineId == null || engineId.trim().isEmpty()) { - engineId = findFirstModelEngine(user); - } - if (engineId == null || engineId.trim().isEmpty()) { - throw new IllegalArgumentException( - "No AI model engine is available. Add a model engine connection to use SQL generation."); - } - if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { - throw new IllegalArgumentException("Model engine does not exist or user does not have access."); - } - - IModelEngine modelEngine = Utility.getModel(engineId); - if (modelEngine == null) { - throw new IllegalArgumentException("Model engine could not be loaded."); - } - - // Build schema section - String schema = buildSchemaForEngine(databaseId); - - // Call LLM - String prompt = SYSTEM_PROMPT + schema + "\n## Request\n" + description.trim(); - Map paramMap = new HashMap<>(); - paramMap.put("use_history", false); - - Map response; - try { - response = modelEngine.ask(prompt, null, this.insight, paramMap).toMap(); - } catch (Exception e) { - classLogger.error("LLM call failed for GenerateSQL on database {}", databaseId, e); - throw new RuntimeException("SQL generation failed: " + e.getMessage(), e); - } - - String sql = extractResponseText(response); - if (sql == null || sql.isBlank()) { - throw new IllegalStateException("The AI model did not return a response. Try again."); - } - sql = stripCodeFences(sql).trim(); - - return new NounMetadata(sql, PixelDataType.CONST_STRING, PixelOperationType.OPERATION); - } - - private static String buildSchemaForEngine(String engineId) { - StringBuilder sb = new StringBuilder(); - try { - IDatabaseEngine dbEngine = Utility.getDatabase(engineId); - if (!(dbEngine instanceof IRDBMSEngine rdbms)) { - return "(non-relational engine — no table schema available)\n"; - } - List tables = rdbms.getPixelConcepts(); - if (tables == null || tables.isEmpty()) { - return "(no tables found)\n"; - } - for (String table : tables) { - List columns = rdbms.getPixelSelectors(table); - sb.append(table).append(" ("); - if (columns != null && !columns.isEmpty()) { - StringBuilder cols = new StringBuilder(); - for (String col : columns) { - String colName = col.contains("__") ? col.split("__")[1] : col; - String dataType = null; - try { - String physUri = rdbms.getPhysicalUriFromPixelSelector(col); - if (physUri != null) dataType = rdbms.getDataTypes(physUri); - } catch (Exception ignored) { } - if (cols.length() > 0) cols.append(", "); - cols.append(colName); - if (dataType != null && !dataType.isBlank()) { - cols.append(": ").append(dataType.trim()); - } - } - sb.append(cols); - } - sb.append(")\n"); - } - } catch (Exception e) { - classLogger.warn("Failed to fetch schema for engine {}", engineId, e); - return "(schema unavailable)\n"; - } - return sb.toString(); - } - - private static String findFirstModelEngine(User user) { - try { - List> engines = SecurityEngineUtils.getUserEngineList( - user, List.of("MODEL"), null, false, null, null, null, "1", "0", null); - if (engines != null && !engines.isEmpty()) { - Object id = engines.get(0).get("database_id"); - return id != null ? String.valueOf(id) : null; - } - } catch (Exception e) { - classLogger.warn("Failed to auto-discover model engine for GenerateSQL", e); - } - return null; - } - - private static String extractResponseText(Map response) { - if (response == null) return null; - Object resp = response.get("response"); - if (resp instanceof String s && !s.isBlank()) return s; - Object output = response.get("output"); - if (output instanceof String s && !s.isBlank()) return s; - Object content = response.get("content"); - if (content instanceof String s && !s.isBlank()) return s; - return null; - } - - private static String stripCodeFences(String raw) { - String trimmed = raw.strip(); - if (trimmed.startsWith("```")) { - int firstNewline = trimmed.indexOf('\n'); - if (firstNewline != -1) trimmed = trimmed.substring(firstNewline + 1); - if (trimmed.endsWith("```")) trimmed = trimmed.substring(0, trimmed.lastIndexOf("```")).strip(); - } - return trimmed; - } - - @Override - public String getReactorDescription() { - return "Generates a SQL SELECT query from a plain-English description using the database schema and an AI model."; - } - - @Override - protected String getDescriptionForKey(String key) { - if (ReactorKeysEnum.DATABASE.getKey().equals(key)) return "ID of the database engine to query."; - if (AutomationConstants.DOC_DESCRIPTION.equals(key)) return "Plain-English description of what data to retrieve (base64-encoded)."; - if (ReactorKeysEnum.ENGINE.getKey().equals(key)) return "Optional model engine ID. Defaults to the first available MODEL engine."; - return super.getDescriptionForKey(key); - } -} diff --git a/src/prerna/reactor/automation/GetAutomationReactor.java b/src/prerna/reactor/automation/GetAutomationReactor.java index bf2170f8eaf..ffef6b63100 100644 --- a/src/prerna/reactor/automation/GetAutomationReactor.java +++ b/src/prerna/reactor/automation/GetAutomationReactor.java @@ -38,8 +38,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.google.gson.reflect.TypeToken; - import prerna.auth.utils.SecurityProjectUtils; import prerna.project.api.IProject; import prerna.reactor.AbstractReactor; @@ -72,6 +70,7 @@ public class GetAutomationReactor extends AbstractReactor { public GetAutomationReactor() { this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey() }; + this.keyRequired = new int[] { 1 }; } @Override @@ -110,7 +109,7 @@ public NounMetadata execute() { try { String json = Files.readString(automationFile.toPath(), StandardCharsets.UTF_8); Map doc = AutomationExecutionUtils.GSON.fromJson(json, - new TypeToken>() {}.getType()); + AutomationExecutionUtils.MAP_TYPE); return new NounMetadata(doc, PixelDataType.MAP, PixelOperationType.OPERATION); } catch (IOException e) { classLogger.error("Error reading automation.json for project {}", projectId, e); @@ -122,4 +121,12 @@ public NounMetadata execute() { public String getReactorDescription() { return "Returns the automation pipeline definition (automation.json) for a project; returns an empty graph when none has been saved."; } + + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) { + return "The project ID of the automation to retrieve."; + } + return super.getDescriptionForKey(key); + } } diff --git a/src/prerna/reactor/automation/SaveAutomationConfigReactor.java b/src/prerna/reactor/automation/SaveAutomationConfigReactor.java index 4029be39ffb..21515dff289 100644 --- a/src/prerna/reactor/automation/SaveAutomationConfigReactor.java +++ b/src/prerna/reactor/automation/SaveAutomationConfigReactor.java @@ -42,6 +42,8 @@ import com.google.gson.reflect.TypeToken; import prerna.auth.utils.SecurityProjectUtils; + + import prerna.reactor.AbstractReactor; import prerna.reactor.agent.mcp.MCPUtility; import prerna.sablecc2.om.PixelDataType; @@ -54,8 +56,11 @@ public class SaveAutomationConfigReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(SaveAutomationConfigReactor.class); + private static final java.lang.reflect.Type LIST_OF_MAP_TYPE = + new TypeToken>>() {}.getType(); + public SaveAutomationConfigReactor() { - this.keysToGet = new String[]{ ReactorKeysEnum.PROJECT.getKey(), ReactorKeysEnum.CONFIG.getKey() }; + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), ReactorKeysEnum.CONFIG.getKey() }; } @Override @@ -91,7 +96,7 @@ public NounMetadata execute() { configFile.getParentFile().mkdirs(); Files.writeString(configFile.toPath(), config, StandardCharsets.UTF_8); } catch (IOException e) { - classLogger.error("Error saving automation config", e); + classLogger.error("Error saving automation config for project {}", projectId, e); throw new IllegalArgumentException("Unable to save automation config: " + e.getMessage()); } @@ -112,10 +117,10 @@ private String restoreMaskedSensitiveValues(String incomingJson, File existingFi } try { List> incoming = AutomationExecutionUtils.GSON.fromJson(incomingJson, - new TypeToken>>() {}.getType()); + LIST_OF_MAP_TYPE); String existingJson = Files.readString(existingFile.toPath(), StandardCharsets.UTF_8); List> existing = AutomationExecutionUtils.GSON.fromJson(existingJson, - new TypeToken>>() {}.getType()); + LIST_OF_MAP_TYPE); if (incoming == null || existing == null || existing.isEmpty()) { return incomingJson; } @@ -154,6 +159,16 @@ public String getReactorDescription() { return "Saves the automation config (key/value env vars and secrets) for a project."; } + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) { + return "The project ID whose automation config should be saved."; + } else if (ReactorKeysEnum.CONFIG.getKey().equals(key)) { + return "Base64-encoded JSON array of config entries (key, value, sensitive flag)."; + } + return super.getDescriptionForKey(key); + } + @Override public Map getMcpToolMetadata() { Map meta = new HashMap<>(); diff --git a/src/prerna/reactor/automation/SaveAutomationReactor.java b/src/prerna/reactor/automation/SaveAutomationReactor.java index 8516d646681..c9260c6c2e0 100644 --- a/src/prerna/reactor/automation/SaveAutomationReactor.java +++ b/src/prerna/reactor/automation/SaveAutomationReactor.java @@ -94,10 +94,17 @@ public NounMetadata execute() { automationFile.getParentFile().mkdirs(); Files.writeString(automationFile.toPath(), json, StandardCharsets.UTF_8); } catch (IOException e) { - classLogger.error("Error saving automation JSON", e); + classLogger.error("Error saving automation JSON for project {}", projectId, e); throw new IllegalArgumentException("Unable to save automation: " + e.getMessage()); } + if (project == null) { + classLogger.warn("Project {} not found in registry - skipping git commit and cluster push", projectId); + SecurityProjectUtils.updateProjectLastEditedDate(projectId); + AutomationMcpSync.syncTriggerAutomationTool(null, projectId, this.insight.getUser(), json); + return new NounMetadata(true, PixelDataType.BOOLEAN, PixelOperationType.OPERATION); + } + List files = new ArrayList<>(); files.add(automationFile.getAbsolutePath()); String versionFolder = AssetUtility.getProjectVersionFolder(project.getProjectName(), projectId); @@ -130,10 +137,20 @@ public String getReactorDescription() { return "Saves the automation graph (automation.json) for a project."; } + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) { + return "The project ID of the automation to save."; + } else if (ReactorKeysEnum.JSON.getKey().equals(key)) { + return "Base64-encoded JSON document representing the automation graph (nodes, edges, transforms)."; + } + return super.getDescriptionForKey(key); + } + @Override public Map getMcpToolMetadata() { Map meta = new HashMap<>(); - // Overwrites the saved automation graph and commits it to source control — + // Overwrites the saved automation graph and commits it to source control - // requires explicit human confirmation. meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); return meta; diff --git a/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java index 285a6e8e952..514a5aa89be 100644 --- a/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java @@ -70,8 +70,8 @@ public Object execute(AutomationNodeContext ctx) throws Exception { Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String pixel = required(config, AutomationConstants.CONFIG_PIXEL, nodeLabel); - String appId = optional(config, AutomationConstants.CONFIG_APP_ID); + String pixel = NodeConfigHelper.required(config, AutomationConstants.CONFIG_PIXEL, nodeLabel); + String appId = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_APP_ID); String resolvedPixel = AutomationExecutionUtils.resolve(pixel, scope, configMap); String resolvedAppId = appId != null ? AutomationExecutionUtils.resolve(appId, scope, configMap) : null; @@ -100,17 +100,4 @@ public Object execute(AutomationNodeContext ctx) throws Exception { return PixelExecutionUtils.runAndCollect(ctx.insight(), resolvedPixel); } - private static String required(Map config, String key, String nodeLabel) { - Object v = config.get(key); - if (v == null || v.toString().isBlank()) { - throw new IllegalArgumentException( - "App node \"" + nodeLabel + "\": '" + key + "' is required"); - } - return v.toString(); - } - - private static String optional(Map config, String key) { - Object v = config.get(key); - return (v == null || v.toString().isBlank()) ? null : v.toString(); - } } diff --git a/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java index 1d33765e188..6dfb6e3e937 100644 --- a/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java @@ -60,10 +60,10 @@ public Object execute(AutomationNodeContext ctx) throws Exception { Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String engineId = required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); - String sql = required(config, AutomationConstants.CONFIG_EXPRESSION, nodeLabel); - String operation = optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_READ); - int limit = optionalInt(config, AutomationConstants.CONFIG_LIMIT, AutomationConstants.DEFAULT_DB_QUERY_LIMIT); + String engineId = NodeConfigHelper.required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); + String sql = NodeConfigHelper.required(config, AutomationConstants.CONFIG_EXPRESSION, nodeLabel); + String operation = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_READ); + int limit = NodeConfigHelper.optionalInt(config, AutomationConstants.CONFIG_LIMIT, AutomationConstants.DEFAULT_DB_QUERY_LIMIT); String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); String resolvedSql = AutomationExecutionUtils.resolve(sql, scope, configMap); @@ -81,23 +81,4 @@ public Object execute(AutomationNodeContext ctx) throws Exception { return PixelExecutionUtils.runAndCollect(ctx.insight(), pixel, timeout); } - private static String required(Map config, String key, String nodeLabel) { - Object v = config.get(key); - if (v == null || v.toString().isBlank()) { - throw new IllegalArgumentException("Database-engine node \"" + nodeLabel + "\": '" + key + "' is required"); - } - return v.toString(); - } - - private static String optional(Map config, String key, String def) { - Object v = config.get(key); - return (v == null || v.toString().isBlank()) ? def : v.toString(); - } - - private static int optionalInt(Map config, String key, int def) { - Object v = config.get(key); - if (v == null) return def; - try { return Integer.parseInt(v.toString().trim()); } - catch (NumberFormatException e) { return def; } - } } diff --git a/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java index 1cb06b7b276..2d7b09f41ae 100644 --- a/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java @@ -52,8 +52,8 @@ public Object execute(AutomationNodeContext ctx) throws Exception { Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String engineId = required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); - String params = optional(config, AutomationConstants.CONFIG_PARAMS, AutomationConstants.EMPTY_JSON_OBJECT); + String engineId = NodeConfigHelper.required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); + String params = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_PARAMS, AutomationConstants.EMPTY_JSON_OBJECT); String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); String resolvedParams = AutomationExecutionUtils.resolve(params, scope, configMap); @@ -86,16 +86,4 @@ private static Map parseParams(String json, String nodeLabel) { } } - private static String required(Map config, String key, String nodeLabel) { - Object v = config.get(key); - if (v == null || v.toString().isBlank()) { - throw new IllegalArgumentException("Function-engine node \"" + nodeLabel + "\": '" + key + "' is required"); - } - return v.toString(); - } - - private static String optional(Map config, String key, String def) { - Object v = config.get(key); - return (v == null || v.toString().isBlank()) ? def : v.toString(); - } } diff --git a/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java index 5a53a6d8547..9b890de637c 100644 --- a/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java @@ -34,8 +34,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.google.gson.reflect.TypeToken; - import prerna.auth.utils.SecurityEngineUtils; import prerna.auth.utils.SecurityQueryUtils; import prerna.engine.api.IModelEngine; @@ -45,6 +43,10 @@ import prerna.reactor.automation.AutomationExecutionUtils; import prerna.util.Utility; +/** + * Executes a model-engine automation node. Supports LLM ({@code llm}) and embeddings + * ({@code embeddings}) operations via the platform {@link prerna.engine.api.IModelEngine} API. + */ public final class ModelEngineNodeExecutor implements IAutomationNodeExecutor { private static final Logger classLogger = LogManager.getLogger(ModelEngineNodeExecutor.class); @@ -56,8 +58,8 @@ public Object execute(AutomationNodeContext ctx) throws Exception { Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String engineId = required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); - String operation = optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_LLM); + String engineId = NodeConfigHelper.required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); + String operation = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_LLM); String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); resolvedEngineId = SecurityQueryUtils.testUserEngineIdForAlias(ctx.insight().getUser(), resolvedEngineId); @@ -74,7 +76,7 @@ public Object execute(AutomationNodeContext ctx) throws Exception { classLogger.debug("Model-engine node \"{}\" executing operation={} via engine {}", nodeLabel, operation, resolvedEngineId); switch (operation) { case AutomationConstants.OP_EMBEDDINGS: { - String values = required(config, AutomationConstants.CONFIG_VALUES, nodeLabel); + String values = NodeConfigHelper.required(config, AutomationConstants.CONFIG_VALUES, nodeLabel); String resolvedValues = AutomationExecutionUtils.resolve(values, scope, configMap); List valueList = Arrays.asList(resolvedValues.split(",")); EmbeddingsModelEngineResponse response = engine.embeddings(valueList, ctx.insight(), null); @@ -82,9 +84,9 @@ public Object execute(AutomationNodeContext ctx) throws Exception { } default: { // llm (and vision/ner as fallback — both use ask() with the primary command field) - String command = required(config, AutomationConstants.CONFIG_COMMAND, nodeLabel); + String command = NodeConfigHelper.required(config, AutomationConstants.CONFIG_COMMAND, nodeLabel); String resolvedCommand = AutomationExecutionUtils.resolve(command, scope, configMap); - String context = optional(config, AutomationConstants.CONFIG_CONTEXT); + String context = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_CONTEXT); String resolvedContext = (context != null) ? AutomationExecutionUtils.resolve(context, scope, configMap) : null; Map params = parseParams(config, scope, configMap, nodeLabel); @@ -98,32 +100,14 @@ public Object execute(AutomationNodeContext ctx) throws Exception { @SuppressWarnings("unchecked") private static Map parseParams(Map config, Map scope, Map configMap, String nodeLabel) { - String paramValues = optional(config, AutomationConstants.CONFIG_PARAM_VALUES); + String paramValues = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_PARAM_VALUES); if (paramValues == null) return null; String resolved = AutomationExecutionUtils.resolve(paramValues, scope, configMap); try { - return AutomationExecutionUtils.GSON.fromJson(resolved, - new TypeToken>() {}.getType()); + return AutomationExecutionUtils.GSON.fromJson(resolved, AutomationExecutionUtils.MAP_TYPE); } catch (Exception e) { throw new IllegalArgumentException("Model-engine node \"" + nodeLabel + "\": paramValues is not valid JSON: " + e.getMessage(), e); } } - private static String required(Map config, String key, String nodeLabel) { - Object v = config.get(key); - if (v == null || v.toString().isBlank()) { - throw new IllegalArgumentException("Model-engine node \"" + nodeLabel + "\": '" + key + "' is required"); - } - return v.toString(); - } - - private static String optional(Map config, String key) { - Object v = config.get(key); - return (v == null || v.toString().isBlank()) ? null : v.toString(); - } - - private static String optional(Map config, String key, String def) { - Object v = config.get(key); - return (v == null || v.toString().isBlank()) ? def : v.toString(); - } } diff --git a/src/prerna/reactor/automation/nodes/NodeConfigHelper.java b/src/prerna/reactor/automation/nodes/NodeConfigHelper.java new file mode 100644 index 00000000000..ea7930dfe37 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/NodeConfigHelper.java @@ -0,0 +1,100 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +/** + * Shared config-extraction helpers used by all automation node executors. + * + *

Each executor previously contained private copies of {@code required()}, + * {@code optional()}, and {@code optionalInt()} with identical logic. This class + * centralises them so changes (e.g. to error message format) propagate everywhere. + */ +final class NodeConfigHelper { + + private NodeConfigHelper() { + // utility class + } + + /** + * Returns the string value of {@code key} from {@code config}, throwing if absent or blank. + * + * @param config node config map + * @param key config key to look up + * @param nodeLabel display label of the containing node (used in the error message) + */ + static String required(Map config, String key, String nodeLabel) { + Object v = config.get(key); + if (v == null || v.toString().isBlank()) { + throw new IllegalArgumentException( + "Node \"" + nodeLabel + "\": '" + key + "' is required"); + } + return v.toString(); + } + + /** + * Returns the string value of {@code key} from {@code config}, or {@code def} if absent or blank. + * + * @param config node config map + * @param key config key to look up + * @param def value to return when key is absent or blank + */ + static String optional(Map config, String key, String def) { + Object v = config.get(key); + return (v == null || v.toString().isBlank()) ? def : v.toString(); + } + + /** + * Returns the string value of {@code key} from {@code config}, or {@code null} if absent or blank. + * + * @param config node config map + * @param key config key to look up + */ + static String optional(Map config, String key) { + return optional(config, key, null); + } + + /** + * Returns the integer value of {@code key} from {@code config}, or {@code def} if absent, blank, + * or not parseable as an integer. + * + * @param config node config map + * @param key config key to look up + * @param def default value when key is absent, blank, or unparseable + */ + static int optionalInt(Map config, String key, int def) { + Object v = config.get(key); + if (v == null) return def; + try { + return Integer.parseInt(v.toString().trim()); + } catch (NumberFormatException e) { + return def; + } + } +} diff --git a/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java index d5d753b9425..2734e062eac 100644 --- a/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java @@ -52,8 +52,8 @@ public Object execute(AutomationNodeContext ctx) throws Exception { Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String engineId = required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); - String operation = optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_LIST); + String engineId = NodeConfigHelper.required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); + String operation = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_LIST); String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); resolvedEngineId = SecurityQueryUtils.testUserEngineIdForAlias(ctx.insight().getUser(), resolvedEngineId); @@ -75,36 +75,36 @@ public Object execute(AutomationNodeContext ctx) throws Exception { classLogger.debug("Storage-engine node \"{}\" executing operation={} via engine {}", nodeLabel, operation, resolvedEngineId); switch (operation) { case AutomationConstants.OP_DOWNLOAD: { - String storagePath = required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); - String filePath = required(config, AutomationConstants.CONFIG_FILE_PATH, nodeLabel); + String storagePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); + String filePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_FILE_PATH, nodeLabel); String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); String resolvedFile = AutomationExecutionUtils.resolve(filePath, scope, configMap); engine.copyToLocal(resolvedStorage, resolvedFile); return "Downloaded: " + resolvedStorage; } case AutomationConstants.OP_UPLOAD: { - String storagePath = required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); - String filePath = required(config, AutomationConstants.CONFIG_FILE_PATH, nodeLabel); + String storagePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); + String filePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_FILE_PATH, nodeLabel); String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); String resolvedFile = AutomationExecutionUtils.resolve(filePath, scope, configMap); engine.copyToStorage(resolvedFile, resolvedStorage, null); return "Uploaded: " + resolvedFile; } case AutomationConstants.OP_DELETE: { - String storagePath = required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); + String storagePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); engine.deleteFromStorage(resolvedStorage); return "Deleted: " + resolvedStorage; } case AutomationConstants.OP_READ_BASE64: { - String storagePath = required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); + String storagePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); byte[] bytes = engine.readBlobToMemory(resolvedStorage); return Base64.getEncoder().encodeToString(bytes); } default: { // list - String storagePath = optional(config, AutomationConstants.CONFIG_STORAGE_PATH, AutomationConstants.DEFAULT_STORAGE_PATH); + String storagePath = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_STORAGE_PATH, AutomationConstants.DEFAULT_STORAGE_PATH); String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); List files = engine.list(resolvedStorage); return files; @@ -112,16 +112,4 @@ public Object execute(AutomationNodeContext ctx) throws Exception { } } - private static String required(Map config, String key, String nodeLabel) { - Object v = config.get(key); - if (v == null || v.toString().isBlank()) { - throw new IllegalArgumentException("Storage-engine node \"" + nodeLabel + "\": '" + key + "' is required"); - } - return v.toString(); - } - - private static String optional(Map config, String key, String def) { - Object v = config.get(key); - return (v == null || v.toString().isBlank()) ? def : v.toString(); - } } diff --git a/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java index ed8c1a9463e..73bc450edce 100644 --- a/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java @@ -52,8 +52,8 @@ public Object execute(AutomationNodeContext ctx) throws Exception { Map scope = ctx.scope(); Map configMap = ctx.configMap(); - String engineId = required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); - String operation = optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_SEARCH); + String engineId = NodeConfigHelper.required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); + String operation = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_SEARCH); String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); resolvedEngineId = SecurityQueryUtils.testUserEngineIdForAlias(ctx.insight().getUser(), resolvedEngineId); @@ -77,7 +77,7 @@ public Object execute(AutomationNodeContext ctx) throws Exception { switch (operation) { case AutomationConstants.OP_ADD_FILE: case AutomationConstants.OP_ADD_CSV: { - String filePaths = required(config, AutomationConstants.CONFIG_FILE_PATH, nodeLabel); + String filePaths = NodeConfigHelper.required(config, AutomationConstants.CONFIG_FILE_PATH, nodeLabel); String resolvedPaths = AutomationExecutionUtils.resolve(filePaths, scope, configMap); List paths = Arrays.asList(resolvedPaths.split(",")); engine.addDocument(paths, null); @@ -88,7 +88,7 @@ public Object execute(AutomationNodeContext ctx) throws Exception { return docs; } case AutomationConstants.OP_DELETE: { - String fileNames = required(config, AutomationConstants.CONFIG_FILE_NAMES, nodeLabel); + String fileNames = NodeConfigHelper.required(config, AutomationConstants.CONFIG_FILE_NAMES, nodeLabel); String resolvedNames = AutomationExecutionUtils.resolve(fileNames, scope, configMap); List names = Arrays.asList(resolvedNames.split(",")); engine.removeDocument(names, null); @@ -96,32 +96,13 @@ public Object execute(AutomationNodeContext ctx) throws Exception { } default: { // search - String command = required(config, AutomationConstants.CONFIG_COMMAND, nodeLabel); + String command = NodeConfigHelper.required(config, AutomationConstants.CONFIG_COMMAND, nodeLabel); String resolvedCommand = AutomationExecutionUtils.resolve(command, scope, configMap); - int limit = optionalInt(config, AutomationConstants.CONFIG_LIMIT, AutomationConstants.DEFAULT_VECTOR_SEARCH_LIMIT); + int limit = NodeConfigHelper.optionalInt(config, AutomationConstants.CONFIG_LIMIT, AutomationConstants.DEFAULT_VECTOR_SEARCH_LIMIT); List> results = engine.nearestNeighbor(ctx.insight(), resolvedCommand, limit, null); return results; } } } - private static String required(Map config, String key, String nodeLabel) { - Object v = config.get(key); - if (v == null || v.toString().isBlank()) { - throw new IllegalArgumentException("Vector-engine node \"" + nodeLabel + "\": '" + key + "' is required"); - } - return v.toString(); - } - - private static String optional(Map config, String key, String def) { - Object v = config.get(key); - return (v == null || v.toString().isBlank()) ? def : v.toString(); - } - - private static int optionalInt(Map config, String key, int def) { - Object v = config.get(key); - if (v == null) return def; - try { return Integer.parseInt(v.toString().trim()); } - catch (NumberFormatException e) { return def; } - } } diff --git a/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java index ab64225d116..6b609e3b749 100644 --- a/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java @@ -77,6 +77,8 @@ public Object execute(AutomationNodeContext ctx) throws Exception { // for the full duration to complete. int remaining = seconds; while (remaining > 0) { + // isCancelRequested checks the DB; this is intentionally bounded by + // WAIT_CANCEL_CHECK_INTERVAL_SECONDS to avoid thrashing the JDBC pool. if (ctx.cancelFlag().get() || AutomationDatabaseUtility.isCancelRequested(ctx.runId())) { throw new AutomationCancelledException("Wait node \"" + nodeLabel + "\" cancelled"); } From 1e668850a9df2afc21a8e119d213fa41216cd431 Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Thu, 13 Aug 2026 12:37:05 -0400 Subject: [PATCH 21/25] chore: general automation cleanup --- .../pipeline/PipelineInvocationHandler.java | 7 +- src/prerna/reactor/automation/AGENTS.md | 81 +++- .../automation/AutomationAskRoomReactor.java | 22 +- .../AutomationCancelledException.java | 20 +- .../automation/AutomationConstants.java | 2 +- .../automation/AutomationDatabaseUtility.java | 58 ++- .../reactor/automation/AutomationMcpSync.java | 24 +- .../automation/AutomationOwlCreator.java | 132 ++++++ .../automation/AutomationRunEngine.java | 18 +- .../automation/BuildAutomationReactor.java | 23 +- .../CancelAutomationRunReactor.java | 4 +- .../automation/CreateAutomationReactor.java | 2 + .../automation/ExplainAutomationReactor.java | 32 +- .../automation/GenerateAutomationReactor.java | 407 ------------------ .../automation/GenerateNodeLabelReactor.java | 8 +- .../automation/GenerateRunSummaryReactor.java | 8 +- .../GetAutomationConfigReactor.java | 23 +- .../automation/GetAutomationReactor.java | 17 +- .../automation/GetAutomationRunReactor.java | 26 +- .../GetAutomationSchemaReactor.java | 2 + .../GetAutomationStructureReactor.java | 42 +- .../GetReactorSignatureReactor.java | 10 +- .../automation/ListAutomationRunsReactor.java | 17 +- .../QuickEditAutomationReactor.java | 380 ++++++++-------- src/prerna/reactor/automation/README.md | 71 --- .../automation/RunAutomationNodeReactor.java | 13 +- .../SaveAutomationConfigReactor.java | 28 +- .../automation/SaveAutomationReactor.java | 7 +- .../automation/TriggerAutomationReactor.java | 25 +- .../nodes/AppEngineNodeExecutor.java | 9 +- .../nodes/AutomationNodeContext.java | 18 +- .../nodes/AutomationNodeExecutors.java | 51 --- .../nodes/DatabaseEngineNodeExecutor.java | 4 +- .../nodes/FunctionEngineNodeExecutor.java | 24 +- .../nodes/IAutomationNodeExecutor.java | 33 +- .../nodes/ModelEngineNodeExecutor.java | 8 +- .../automation/nodes/NodeConfigHelper.java | 67 +-- .../nodes/StorageEngineNodeExecutor.java | 16 +- .../nodes/VectorEngineNodeExecutor.java | 29 +- .../automation/nodes/WaitNodeExecutor.java | 7 +- .../{ => utils}/AutomationExecutionUtils.java | 328 +++----------- .../utils/AutomationGenerationUtils.java | 334 ++++++++++++++ .../{ => utils}/PixelExecutionUtils.java | 28 +- .../reactor/scheduler/SchedulerConstants.java | 1 - .../scheduler/SchedulerOwlCreator.java | 39 -- src/prerna/util/SMSSWebWatcher.java | 9 +- 46 files changed, 1214 insertions(+), 1300 deletions(-) create mode 100644 src/prerna/reactor/automation/AutomationOwlCreator.java delete mode 100644 src/prerna/reactor/automation/GenerateAutomationReactor.java delete mode 100644 src/prerna/reactor/automation/README.md delete mode 100644 src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java rename src/prerna/reactor/automation/{ => utils}/AutomationExecutionUtils.java (66%) create mode 100644 src/prerna/reactor/automation/utils/AutomationGenerationUtils.java rename src/prerna/reactor/automation/{ => utils}/PixelExecutionUtils.java (85%) diff --git a/src/prerna/engine/impl/pipeline/PipelineInvocationHandler.java b/src/prerna/engine/impl/pipeline/PipelineInvocationHandler.java index bce66f7454d..fb07d15558d 100644 --- a/src/prerna/engine/impl/pipeline/PipelineInvocationHandler.java +++ b/src/prerna/engine/impl/pipeline/PipelineInvocationHandler.java @@ -61,8 +61,6 @@ import com.github.f4b6a3.uuid.alt.GUID; import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import com.google.gson.JsonPrimitive; -import com.google.gson.JsonSerializer; import com.google.gson.ToNumberPolicy; import prerna.engine.api.IEngine; @@ -116,10 +114,7 @@ public class PipelineInvocationHandler implements InvocationHandler { .registerTypeAdapter(ZoneOffset.class, new ZoneOffsetTypeAdapter()) .registerTypeAdapter(Insight.class, new LoggingInsightAdapter()) .registerTypeAdapter(LocalDateTime.class, new LocalDateTimeAdapter()) - .registerTypeAdapter(ZonedDateTime.class, new ZonedDateTimeAdapter()) - .registerTypeHierarchyAdapter(Throwable.class, - (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.toString())) - .create(); + .registerTypeAdapter(ZonedDateTime.class, new ZonedDateTimeAdapter()).create(); private final String REQUEST_NOT_TRACKED = "REQUEST NOT TRACKED"; private final String RESPONSE_NOT_TRACKED = "RESPONSE NOT TRACKED"; diff --git a/src/prerna/reactor/automation/AGENTS.md b/src/prerna/reactor/automation/AGENTS.md index 79c3e918323..f71f51d67f3 100644 --- a/src/prerna/reactor/automation/AGENTS.md +++ b/src/prerna/reactor/automation/AGENTS.md @@ -1,6 +1,78 @@ # Automation Engine — Agent Guide -Read `README.md` first for the execution model and DB schema. +Executes sequential node pipelines against SEMOSS engines. Users build a pipeline in the form editor (FE), save it as `automation.json`, then trigger it manually. Each run is tracked in the DB and the FE polls for progress. + +## How it works + +``` +FE calls runPixelAsync("TriggerAutomation(...)") + → Monolith spawns a virtual thread, returns jobId immediately + → FE polls GetActiveAutomationRun every 500ms (up to 10×) to get runId + +TriggerAutomationReactor (virtual thread, synchronous) + → reads automation.json (nodes in order) + → claims single-run slot (AUTOMATION_ACTIVE_RUN) → runId visible to FE + → inserts AUTOMATION_RUNS + AUTOMATION_NODE_OUTPUTS rows + → calls AutomationRunEngine.run() synchronously + → returns completed run result to jobId slot when done + +AutomationRunEngine (same virtual thread) + → iterates nodes in saved order + → dispatches each node to its IAutomationNodeExecutor + → writes node output + status to AUTOMATION_NODE_OUTPUTS after each node + → FE polls GetAutomationRun every 3s while runId is known +``` + +## Reactors + +| Reactor | Pixel | What it does | +| --- | --- | --- | +| `TriggerAutomationReactor` | `TriggerAutomation(project=["id"])` | Starts a run synchronously; returns completed run result | +| `GetActiveAutomationRunReactor` | `GetActiveAutomationRun(project=["id"])` | Returns `{RUN_ID, PROJECT_ID}` from active-run lock table; empty map when idle | +| `GetAutomationReactor` | `GetAutomation(project=["id"])` | Returns saved pipeline definition (automation.json) | +| `GetAutomationConfigReactor` | `GetAutomationConfig(project=["id"])` | Returns env var/secret config; masks sensitive values | +| `GetAutomationRunReactor` | `GetAutomationRun(project=["id"], runId=["id"])` | Returns live run state for FE polling | +| `ListAutomationRunsReactor` | `ListAutomationRuns(project=["id"])` | Returns run history | +| `CancelAutomationRunReactor` | `CancelAutomationRun(project=["id"], runId=["id"])` | Cancels an in-progress run | +| `SaveAutomationReactor` | `SaveAutomation(project=["id"], config=["{}"])` | Persists pipeline definition | +| `SaveAutomationConfigReactor` | `SaveAutomationConfig(project=["id"], config=["[]"])` | Persists env var config | +| `RunAutomationNodeReactor` | `RunAutomationNode(project=["id"], nodeId=["id"])` | Single-node test run; result not persisted | + +## Node types + +Each node type has a corresponding `IAutomationNodeExecutor` in `nodes/`: + +| Type | Executor | What it does | +| --- | --- | --- | +| `trigger` | (no executor) | Seed node; provides `triggered_at`, `date`, `run_id` scope vars | +| `database-engine` | `DatabaseEngineNodeExecutor` | Runs SQL via `SqlQuery` pixel | +| `model-engine` | `ModelEngineNodeExecutor` | LLM ask or embeddings via `IModelEngine` | +| `vector-engine` | `VectorEngineNodeExecutor` | Search, add, delete, list via `IVectorDatabaseEngine` | +| `storage-engine` | `StorageEngineNodeExecutor` | File operations via `IStorageEngine` | +| `function-engine` | `FunctionEngineNodeExecutor` | Function invocation via `IFunctionEngine` | +| `app` | `AppEngineNodeExecutor` | Arbitrary pixel, optionally scoped to a project | +| `wait` | `WaitNodeExecutor` | Sleep N seconds; cancel-aware | + +## DB tables + +| Table | Purpose | +| --- | --- | +| `AUTOMATION_ACTIVE_RUN` | PK on `PROJECT_ID` — enforces one concurrent run per project | +| `AUTOMATION_RUNS` | One row per run: status, timing, node counts | +| `AUTOMATION_NODE_OUTPUTS` | One row per node per run: status, output, preview, duration | + +## Key classes + +| Class | Purpose | +| --- | --- | +| `AutomationRunEngine` | Orchestrates a full pipeline run end-to-end | +| `AutomationExecutionUtils` | Shared statics: GSON, scope building, variable resolution, output transforms, preview generation | +| `AutomationGenerationUtils` | LLM/generation helpers: engine discovery, prompt building, response extraction | +| `AutomationDatabaseUtility` | All DB reads/writes for runs and node outputs | +| `PixelExecutionUtils` | Timeout-enforced pixel execution with ThreadStore propagation | +| `AutomationConstants` | String constants for all keys, statuses, and file names | + +--- ## Adding a new node type @@ -11,7 +83,7 @@ Read `README.md` first for the execution model and DB schema. - Throw `IllegalArgumentException` for missing required config fields - Use `AutomationExecutionUtils.GSON` — do not declare a local `Gson` instance -2. **Register it** in `AutomationNodeExecutors.EXECUTORS` map +2. **Register it** in the `EXECUTORS` map on `IAutomationNodeExecutor` (static field on the interface; use `Map.ofEntries` if adding an 11th entry — `Map.of` only supports 10) 3. **Add the type constant** to `AutomationConstants` (e.g. `NODE_TYPE_FOO = "foo-engine"`) @@ -46,7 +118,7 @@ Use the shared instance — never declare your own: ```java // ✅ -AutomationExecutionUtils.GSON.fromJson(json, new TypeToken>() {}.getType()); +AutomationExecutionUtils.GSON.fromJson(json, AutomationExecutionUtils.MAP_TYPE); // ❌ private static final Gson GSON = new GsonBuilder().create(); @@ -57,7 +129,8 @@ private static final Gson GSON = new GsonBuilder().create(); - Keep reactors thin: parse params, auth-check, delegate, return. Business logic belongs in `AutomationRunEngine`, `AutomationExecutionUtils`, or an executor. - Always call `organizeKeys()` before reading `this.keyValue` - Use `SecurityProjectUtils.testUserProjectIdForAlias` to resolve alias → UUID before any lookup -- Add `getReactorDescription()` to every reactor +- Add `getReactorDescription()` and `getDescriptionForKey()` to every reactor +- MCP-destructive reactors (save, trigger, cancel) must override `getMcpToolMetadata()` to `MCPExecution.ASK` ## What not to change diff --git a/src/prerna/reactor/automation/AutomationAskRoomReactor.java b/src/prerna/reactor/automation/AutomationAskRoomReactor.java index 11c0a4b5d8f..d29ddddf7f0 100644 --- a/src/prerna/reactor/automation/AutomationAskRoomReactor.java +++ b/src/prerna/reactor/automation/AutomationAskRoomReactor.java @@ -27,6 +27,8 @@ *******************************************************************************/ package prerna.reactor.automation; +import prerna.reactor.automation.utils.AutomationGenerationUtils; + import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Map; @@ -68,24 +70,24 @@ public class AutomationAskRoomReactor extends AbstractReactor { private static final String SYSTEM_PROMPT = "You are an AI assistant that helps users design automation workflows on a no-code platform. " + "Workflows are linear sequences of steps: database queries, AI model calls, file storage, vector search, or custom functions. " - + "Only manual triggers exist — do not ask about scheduling.\n\n" + + "Only manual triggers exist - do not ask about scheduling.\n\n" + "You have access to tools. Use them to help the user (for example, query a database to understand its structure " + "so you can give accurate step descriptions).\n\n" + "CONVERSATION PHASES:\n\n" - + "Phase 1 — Gather requirements (at most 1-2 short questions, one at a time):\n" + + "Phase 1 - Gather requirements (at most 1-2 short questions, one at a time):\n" + "- Ask what the automation should do if unclear.\n" + "- Ask where results should go, or one other essential clarification.\n" + "- Keep responses under 50 words per question.\n\n" - + "Phase 2 — Plan + build signal (in ONE response, once you have enough info):\n" + + "Phase 2 - Plan + build signal (in ONE response, once you have enough info):\n" + "- Present a concise numbered plain-English plan.\n" + "- On the very next line after the plan, output the build signal JSON:\n" + " {\"action\":\"build\",\"description\":\"\"}\n" + "- The description must include all steps in enough detail for an AI to build them.\n" - + "- Do NOT ask 'does this look right?' — include the build signal in the same response as the plan.\n\n" - + "Phase 3 — If the user requests changes after seeing the plan:\n" + + "- Do NOT ask 'does this look right?' - include the build signal in the same response as the plan.\n\n" + + "Phase 3 - If the user requests changes after seeing the plan:\n" + "- Acknowledge in one short sentence.\n" + "- Immediately output the revised plan + a new build signal in the same response.\n" - + "- Do NOT narrate what you will change — just show the revised plan and the signal.\n\n" + + "- Do NOT narrate what you will change - just show the revised plan and the signal.\n\n" + "RULES: Never mention engine IDs, node types, or JSON structure to the user. Be concise."; public AutomationAskRoomReactor() { @@ -116,7 +118,7 @@ public NounMetadata execute() { throw new IllegalArgumentException("command must not be empty."); } - String engineId = AutomationExecutionUtils.findFirstModelEngine(user); + String engineId = AutomationGenerationUtils.findFirstModelEngine(user); if (engineId == null || engineId.isBlank()) { throw new IllegalArgumentException( "No AI model engine is available. Add a model engine connection to use this feature."); @@ -131,7 +133,7 @@ public NounMetadata execute() { roomId = "automationchat" + projectId.replace("-", "").substring(0, Math.min(8, projectId.replace("-", "").length())); } - Map options = AutomationExecutionUtils.buildEngineMcpOptions(user, SYSTEM_PROMPT); + Map options = AutomationGenerationUtils.buildEngineMcpOptions(user, SYSTEM_PROMPT); RoomUtils.createRoomIfNotExists(roomId, this.insight, modelEngine, command, null, options, null, projectId, null); @@ -151,7 +153,7 @@ public NounMetadata execute() { String status = (String) result.get("status"); if ("FAILED".equals(status)) { String errMsg = (String) result.get("errorMessage"); - classLogger.error("AutomationAskRoom run failed: project={} error={}", projectId, errMsg); + classLogger.error("AutomationAskRoom run failed: project={} error={}", projectId, (errMsg != null ? errMsg : "unknown error")); throw new RuntimeException("Chat failed: " + (errMsg != null ? errMsg : "unknown error")); } @@ -182,7 +184,7 @@ private static String decodeCommand(String raw) { @Override public String getReactorDescription() { return "Room-aware conversational AI for designing automation workflows. " - + "Uses the platform RunAgent harness with MCP tool access — the model can query " + + "Uses the platform RunAgent harness with MCP tool access - the model can query " + "databases, search vectors, and use other engines during the conversation. " + "History is managed server-side. " + "Signals build-readiness via: {\"action\":\"build\",\"description\":\"...\"}"; diff --git a/src/prerna/reactor/automation/AutomationCancelledException.java b/src/prerna/reactor/automation/AutomationCancelledException.java index c74940fe740..f01c68f361d 100644 --- a/src/prerna/reactor/automation/AutomationCancelledException.java +++ b/src/prerna/reactor/automation/AutomationCancelledException.java @@ -27,17 +27,21 @@ *******************************************************************************/ package prerna.reactor.automation; +import prerna.reactor.automation.utils.PixelExecutionUtils; + /** - * Thrown mid-node when a cancellation request is detected during a blocking operation. + * Compatibility shim — extends {@link PixelExecutionUtils.AutomationCancelledException} so that + * existing callers importing this top-level class continue to compile unmodified. + * + *

The canonical definition is now the inner class + * {@link PixelExecutionUtils.AutomationCancelledException}. This file will be removed in a later + * cleanup pass once all callers (in particular the {@code nodes/} sub-package files) are updated + * to reference the inner class directly. * - *

Using a distinct unchecked exception type (rather than a flag return value or a checked - * exception) lets nodes that loop internally - such as {@code WaitNodeExecutor} sleeping in - * chunks - abort cleanly without threading a cancellation result through every call frame. The - * caller ({@code TriggerAutomationReactor.executeSingleNode}) catches this type specifically and - * records the run as {@link AutomationConstants#STATUS_CANCELLED} instead of - * {@link AutomationConstants#STATUS_FAILED}, so the end-user sees the correct terminal state. + * @deprecated Use {@link PixelExecutionUtils.AutomationCancelledException} directly. */ -public class AutomationCancelledException extends RuntimeException { +@Deprecated +public class AutomationCancelledException extends PixelExecutionUtils.AutomationCancelledException { private static final long serialVersionUID = 1L; diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index 1e9094b5ffc..7b3284892c7 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -229,7 +229,7 @@ private AutomationConstants() {} public static final String VARCHAR_50 = "VARCHAR(50)"; public static final String VARCHAR_255 = "VARCHAR(255)"; public static final String VARCHAR_500 = "VARCHAR(500)"; - public static final String VARCHAR_2000 = "VARCHAR(2000)"; + public static final String VARCHAR_2000 = "VARCHAR (2000)"; public static final String INTEGER = "INTEGER"; public static final String BIGINT = "BIGINT"; public static final String NOT_NULL = "NOT NULL"; diff --git a/src/prerna/reactor/automation/AutomationDatabaseUtility.java b/src/prerna/reactor/automation/AutomationDatabaseUtility.java index 4f6c7305a9b..781ad6cb3db 100644 --- a/src/prerna/reactor/automation/AutomationDatabaseUtility.java +++ b/src/prerna/reactor/automation/AutomationDatabaseUtility.java @@ -27,6 +27,8 @@ *******************************************************************************/ package prerna.reactor.automation; +import prerna.reactor.automation.utils.AutomationExecutionUtils; + import static prerna.reactor.automation.AutomationConstants.AUTOMATION_ID; import static prerna.reactor.automation.AutomationConstants.BIGINT; import static prerna.reactor.automation.AutomationConstants.CANCEL_REQUESTED; @@ -184,9 +186,10 @@ private AutomationDatabaseUtility() { // -- Initialization ------------------------------------------------------------ /** - * Creates automation tables in the scheduler DB if they don't exist. - * Called at platform startup after the scheduler DB is loaded. - * Safe to call on every startup (uses IF NOT EXISTS / metadata checks). + * Creates automation tables in the scheduler DB if they don't exist, and + * registers them in the OWL. Called at platform startup after the scheduler + * DB is loaded. Safe to call on every startup (uses IF NOT EXISTS / metadata + * checks). */ public static void initialize() { IRDBMSEngine schedulerDb = getSchedulerDb(); @@ -195,6 +198,18 @@ public static void initialize() { return; } + // Register the automation OWL schema in the scheduler DB if any tables or + // columns are missing. This keeps the OWL declaration entirely within this + // package rather than depending on SchedulerOwlCreator. + AutomationOwlCreator owlCreator = new AutomationOwlCreator(); + if (owlCreator.needsRemake(schedulerDb)) { + try { + owlCreator.remakeOwl(schedulerDb); + } catch (Exception e) { + classLogger.error("Failed to update automation OWL schema in scheduler DB", e); + } + } + Connection conn = null; try { conn = schedulerDb.getConnection(); @@ -876,6 +891,43 @@ public static List> getNodeOutputsForRun(String runId) { return results != null ? results : new ArrayList<>(); } + // -- Result Assembly ----------------------------------------------------------- + + /** + * Builds a list of per-node result maps from the raw DB output rows returned by + * {@link #getNodeOutputsForRun(String)}. The shape matches what + * {@link prerna.reactor.automation.GetAutomationRunReactor} and + * {@link prerna.reactor.automation.TriggerAutomationReactor} return to callers. + * + *

Each entry contains: nodeId, nodeLabel, status, durationMs, outputPreview + * (falls back from outputValue when blank), outputValue, and errorMessage. + * + * @param nodeOutputs ordered rows from {@link #getNodeOutputsForRun(String)} + * @return mutable list of node result maps (empty when {@code nodeOutputs} is null) + */ + public static List> buildNodeResults(List> nodeOutputs) { + List> nodeResults = new ArrayList<>(); + if (nodeOutputs == null) { + return nodeResults; + } + for (Map output : nodeOutputs) { + Map nodeResult = new java.util.HashMap<>(); + nodeResult.put(AutomationConstants.NODE_ID, output.get(AutomationConstants.NODE_ID)); + nodeResult.put(AutomationConstants.NODE_LABEL, output.get(AutomationConstants.NODE_LABEL)); + nodeResult.put(AutomationConstants.STATUS, output.get(AutomationConstants.STATUS)); + nodeResult.put(AutomationConstants.DURATION_MS, output.get(AutomationConstants.DURATION_MS)); + String outputForDisplay = (String) output.get(AutomationConstants.OUTPUT_VALUE); + if (outputForDisplay == null || outputForDisplay.isBlank()) { + outputForDisplay = (String) output.get(AutomationConstants.OUTPUT_PREVIEW); + } + nodeResult.put(AutomationConstants.OUTPUT_PREVIEW, outputForDisplay); + nodeResult.put(AutomationConstants.OUTPUT_VALUE, output.get(AutomationConstants.OUTPUT_VALUE)); + nodeResult.put(AutomationConstants.ERROR_MESSAGE, output.get(AutomationConstants.ERROR_MESSAGE)); + nodeResults.add(nodeResult); + } + return nodeResults; + } + // -- Table Creation ------------------------------------------------------------ private static void createAutomationRunsTable(Connection conn, AbstractSqlQueryUtil queryUtil, diff --git a/src/prerna/reactor/automation/AutomationMcpSync.java b/src/prerna/reactor/automation/AutomationMcpSync.java index e9c48843694..3830e43e412 100644 --- a/src/prerna/reactor/automation/AutomationMcpSync.java +++ b/src/prerna/reactor/automation/AutomationMcpSync.java @@ -27,6 +27,8 @@ *******************************************************************************/ package prerna.reactor.automation; +import prerna.reactor.automation.utils.AutomationExecutionUtils; + import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -117,6 +119,10 @@ public static void syncTriggerAutomationTool(IProject project, String projectId, generated.put(buildBuildAutomationTool(projectId)); MCPUtility.stampGenerator(generated, AUTOMATION_MCP_GENERATOR_ID); + // One-arg form is equivalent to two-arg: it does Utility.getProject(projectId) internally + // and then delegates to getProjectAssetsFolder(projectName, projectId) - same path, no difference. + // We already hold IProject above but the registry lookup is cheap and avoids coupling the + // call site to projectName extraction just for this one call. String assetsFolder = AssetUtility.getProjectAssetsFolder(projectId); String outputFileLoc = Paths.get(assetsFolder, "mcp", "pixel_mcp.json").toString(); JSONArray merged = MCPUtility.mergeGeneratedTools( @@ -159,7 +165,7 @@ private static JSONObject buildTriggerAutomationTool(String projectId, String au + "per-workflow summary once complete (e.g. \"Indexed 20 files\")."; } if (hasDbNodes) { - description += " This automation has database nodes that accept SQL queries - call GetAutomationSchema first" + description += " This automation has database nodes that accept SQL queries - call GetAutomationSchema first" + " to discover the exact table and column names before writing SQL."; } tool.put("description", description); @@ -263,7 +269,7 @@ private static boolean hasPlaygroundDbNodes(String automationJson) { } /** - * Builds the {@code BuildAutomation} MCP tool - lets an agent in Playground generate or edit + * Builds the {@code BuildAutomation} MCP tool - lets an agent in Playground generate or edit * this project's automation from a plain-English description. Execution mode is ASK so the user * can review the generated document before it is saved. */ @@ -274,7 +280,7 @@ private static JSONObject buildBuildAutomationTool(String projectId) { tool.put("description", "Generates or edits this project's automation from a plain-English description. " + "The model iteratively gathers context (database schema, available reactors) before producing a complete automation document. " - + "Does NOT save automatically - call SaveAutomation with the returned JSON to persist. " + + "Does NOT save automatically - call SaveAutomation with the returned JSON to persist. " + "Use currentDoc to pass the existing automation JSON (base64-encoded) for edit mode."); JSONObject projectProp = new JSONObject(); @@ -317,7 +323,7 @@ private static JSONObject buildGetAutomationSchemaTool(String projectId) { tool.put("title", "Get Automation Database Schema"); tool.put("description", "Returns the physical table and column names for database nodes in this automation that accept SQL input. " - + "Call this before TriggerAutomation when you need to write a SQL query - it gives you the exact " + + "Call this before TriggerAutomation when you need to write a SQL query - it gives you the exact " + "table and column names available in the database."); JSONObject projectProp = new JSONObject(); @@ -355,19 +361,19 @@ private static void writeMcpJson(String outputFileLoc, JSONArray tools) throws I } private static String buildPlaygroundParamDescription(String nodeType, String fieldName) { - if ("database-engine".equals(nodeType) && "expression".equals(fieldName)) { + if (AutomationConstants.NODE_DATABASE_ENGINE.equals(nodeType) && AutomationConstants.CONFIG_EXPRESSION.equals(fieldName)) { return "SQL query to execute against the connected database"; } - if ("model-engine".equals(nodeType) && "command".equals(fieldName)) { + if (AutomationConstants.NODE_MODEL_ENGINE.equals(nodeType) && AutomationConstants.CONFIG_COMMAND.equals(fieldName)) { return "Natural language prompt to send to the language model"; } - if ("model-engine".equals(nodeType) && "context".equals(fieldName)) { + if (AutomationConstants.NODE_MODEL_ENGINE.equals(nodeType) && AutomationConstants.CONFIG_CONTEXT.equals(fieldName)) { return "System instructions for the language model's behavior"; } - if ("vector-engine".equals(nodeType) && "command".equals(fieldName)) { + if (AutomationConstants.NODE_VECTOR_ENGINE.equals(nodeType) && AutomationConstants.CONFIG_COMMAND.equals(fieldName)) { return "Search query to run against the vector database"; } - if ("function-engine".equals(nodeType) && "params".equals(fieldName)) { + if (AutomationConstants.NODE_FUNCTION_ENGINE.equals(nodeType) && AutomationConstants.CONFIG_PARAMS.equals(fieldName)) { return "JSON parameters to pass to the function"; } return "Input for the " + fieldName + " field"; diff --git a/src/prerna/reactor/automation/AutomationOwlCreator.java b/src/prerna/reactor/automation/AutomationOwlCreator.java new file mode 100644 index 00000000000..00b7cfa02a1 --- /dev/null +++ b/src/prerna/reactor/automation/AutomationOwlCreator.java @@ -0,0 +1,132 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import static prerna.reactor.automation.AutomationConstants.AUTOMATION_ID; +import static prerna.reactor.automation.AutomationConstants.BIGINT; +import static prerna.reactor.automation.AutomationConstants.CANCEL_REQUESTED; +import static prerna.reactor.automation.AutomationConstants.CLAIMED_AT; +import static prerna.reactor.automation.AutomationConstants.COMPLETED_AT; +import static prerna.reactor.automation.AutomationConstants.COMPLETED_NODES; +import static prerna.reactor.automation.AutomationConstants.CREATED_BY; +import static prerna.reactor.automation.AutomationConstants.DURATION_MS; +import static prerna.reactor.automation.AutomationConstants.ERROR_MESSAGE; +import static prerna.reactor.automation.AutomationConstants.EXECUTION_ORDER; +import static prerna.reactor.automation.AutomationConstants.FAILED_NODE_ID; +import static prerna.reactor.automation.AutomationConstants.INTEGER; +import static prerna.reactor.automation.AutomationConstants.LAST_HEARTBEAT; +import static prerna.reactor.automation.AutomationConstants.NODE_ID; +import static prerna.reactor.automation.AutomationConstants.NODE_LABEL; +import static prerna.reactor.automation.AutomationConstants.OUTPUT_PREVIEW; +import static prerna.reactor.automation.AutomationConstants.OUTPUT_VALUE; +import static prerna.reactor.automation.AutomationConstants.OUTPUT_VAR; +import static prerna.reactor.automation.AutomationConstants.PROJECT_ID; +import static prerna.reactor.automation.AutomationConstants.RESULT_SUMMARY_COL; +import static prerna.reactor.automation.AutomationConstants.RUN_ID; +import static prerna.reactor.automation.AutomationConstants.STARTED_AT; +import static prerna.reactor.automation.AutomationConstants.STATUS; +import static prerna.reactor.automation.AutomationConstants.TABLE_AUTOMATION_ACTIVE_RUN; +import static prerna.reactor.automation.AutomationConstants.TABLE_AUTOMATION_NODE_OUTPUTS; +import static prerna.reactor.automation.AutomationConstants.TABLE_AUTOMATION_RUNS; +import static prerna.reactor.automation.AutomationConstants.TOTAL_NODES; +import static prerna.reactor.automation.AutomationConstants.TRIGGER_TYPE; +import static prerna.reactor.automation.AutomationConstants.VARCHAR_2000; +import static prerna.reactor.automation.AutomationConstants.VARCHAR_255; +import static prerna.reactor.automation.AutomationConstants.VARCHAR_500; + +import java.util.ArrayList; +import java.util.Arrays; + +import org.javatuples.Pair; + +import prerna.engine.impl.owl.AbstractOwlCreator; + +/** + * OWL schema declaration for the three automation tables stored in the + * scheduler database. Follows the same pattern as other system-engine OWL + * creators (e.g. {@link prerna.reactor.scheduler.SchedulerOwlCreator}). + * + *

Called from {@link AutomationDatabaseUtility#initialize()} so that the + * automation OWL schema stays entirely within the {@code prerna.reactor.automation} + * package and does not require the scheduler package to import automation types. + */ +public class AutomationOwlCreator extends AbstractOwlCreator { + + // Reuse the DB-level BOOLEAN/TIMESTAMP/CLOB type names used by the scheduler OWL. + // SchedulerConstants uses these string literals directly (e.g. "BOOLEAN", "TIMESTAMP", "CLOB"). + private static final String BOOLEAN = "BOOLEAN"; + private static final String TIMESTAMP = "TIMESTAMP"; + private static final String CLOB = "CLOB"; + + public AutomationOwlCreator() { + createColumnsAndTypes(); + } + + public void createColumnsAndTypes() { + this.allSchemas = new ArrayList<>(); + + // @formatter:off + addTable(TABLE_AUTOMATION_RUNS, Arrays.asList( + Pair.with(RUN_ID, VARCHAR_255), + Pair.with(PROJECT_ID, VARCHAR_255), + Pair.with(AUTOMATION_ID, VARCHAR_255), + Pair.with(STATUS, VARCHAR_255), + Pair.with(TRIGGER_TYPE, VARCHAR_255), + Pair.with(STARTED_AT, TIMESTAMP), + Pair.with(COMPLETED_AT, TIMESTAMP), + Pair.with(FAILED_NODE_ID, VARCHAR_255), + Pair.with(ERROR_MESSAGE, CLOB), + Pair.with(LAST_HEARTBEAT, TIMESTAMP), + Pair.with(TOTAL_NODES, INTEGER), + Pair.with(COMPLETED_NODES, INTEGER), + Pair.with(CREATED_BY, VARCHAR_255), + Pair.with(CANCEL_REQUESTED, BOOLEAN), + Pair.with(RESULT_SUMMARY_COL, VARCHAR_2000))); + + addTable(TABLE_AUTOMATION_NODE_OUTPUTS, Arrays.asList( + Pair.with(RUN_ID, VARCHAR_255), + Pair.with(NODE_ID, VARCHAR_255), + Pair.with(NODE_LABEL, VARCHAR_500), + Pair.with(EXECUTION_ORDER, INTEGER), + Pair.with(STATUS, VARCHAR_255), + Pair.with(STARTED_AT, TIMESTAMP), + Pair.with(COMPLETED_AT, TIMESTAMP), + Pair.with(DURATION_MS, BIGINT), + Pair.with(OUTPUT_VAR, VARCHAR_255), + Pair.with(OUTPUT_VALUE, CLOB), + Pair.with(OUTPUT_PREVIEW, VARCHAR_2000), + Pair.with(ERROR_MESSAGE, CLOB))); + + addTable(TABLE_AUTOMATION_ACTIVE_RUN, Arrays.asList( + Pair.with(PROJECT_ID, VARCHAR_255), + Pair.with(RUN_ID, VARCHAR_255), + Pair.with(CLAIMED_AT, TIMESTAMP))); + // @formatter:on + } + +} diff --git a/src/prerna/reactor/automation/AutomationRunEngine.java b/src/prerna/reactor/automation/AutomationRunEngine.java index ee532022440..8056d024e21 100644 --- a/src/prerna/reactor/automation/AutomationRunEngine.java +++ b/src/prerna/reactor/automation/AutomationRunEngine.java @@ -27,10 +27,12 @@ *******************************************************************************/ package prerna.reactor.automation; +import prerna.reactor.automation.utils.PixelExecutionUtils; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + import java.sql.Timestamp; import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneOffset; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -47,7 +49,6 @@ import prerna.om.Insight; import prerna.om.ThreadStore; import prerna.reactor.automation.nodes.AutomationNodeContext; -import prerna.reactor.automation.nodes.AutomationNodeExecutors; import prerna.reactor.automation.nodes.IAutomationNodeExecutor; import prerna.sablecc2.comm.PixelJobManager; import prerna.util.Utility; @@ -129,7 +130,7 @@ public static Map run(String runId, String projectId, Map nodeResult; try { nodeResult = executeSingleNode(runId, projectId, node, scope, configMap, cancelled, insight); - } catch (AutomationCancelledException ace) { + } catch (PixelExecutionUtils.AutomationCancelledException ace) { classLogger.info("Automation run {} cancelled during node {} ({})", runId, nodeId, nodeLabel); AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_CANCELLED, nodeId, ace.getMessage()); @@ -225,14 +226,14 @@ private static Map executeSingleNode(String runId, String projec classLogger.debug("Executing node {} ({}) type={} in run {}", nodeId, nodeLabel, type, runId); AutomationDatabaseUtility.markNodeRunning(runId, nodeId); - Timestamp startedAt = toTimestamp(Instant.now()); + Timestamp startedAt = Utility.getSqlTimestampUTC(java.time.LocalDateTime.ofInstant(Instant.now(), java.time.ZoneOffset.UTC)); long startMs = System.currentTimeMillis(); try { AutomationNodeContext ctx = new AutomationNodeContext( runId, projectId, node, scope, configMap, insight, cancelFlag); - IAutomationNodeExecutor executor = AutomationNodeExecutors.EXECUTORS.get(type); + IAutomationNodeExecutor executor = IAutomationNodeExecutor.EXECUTORS.get(type); if (executor == null) { throw new IllegalArgumentException("Unsupported node type: " + type); } @@ -252,7 +253,7 @@ private static Map executeSingleNode(String runId, String projec result.put(AutomationConstants.RESULT_OUTPUT_VALUE, transformed); return result; - } catch (AutomationCancelledException ace) { + } catch (PixelExecutionUtils.AutomationCancelledException ace) { long durationMs = System.currentTimeMillis() - startMs; AutomationDatabaseUtility.updateNodeFailed(runId, nodeId, startedAt, durationMs, ace.getMessage()); throw ace; @@ -299,7 +300,4 @@ private static Map buildNodeResult(String nodeId, String nodeLab return result; } - private static Timestamp toTimestamp(Instant instant) { - return Utility.getSqlTimestampUTC(LocalDateTime.ofInstant(instant, ZoneOffset.UTC)); - } } diff --git a/src/prerna/reactor/automation/BuildAutomationReactor.java b/src/prerna/reactor/automation/BuildAutomationReactor.java index 83f638ba398..a2f8797b55e 100644 --- a/src/prerna/reactor/automation/BuildAutomationReactor.java +++ b/src/prerna/reactor/automation/BuildAutomationReactor.java @@ -27,6 +27,8 @@ *******************************************************************************/ package prerna.reactor.automation; +import prerna.reactor.automation.utils.AutomationGenerationUtils; + import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Map; @@ -151,7 +153,7 @@ public NounMetadata execute() { } if (engineId == null || engineId.trim().isEmpty()) { - engineId = AutomationExecutionUtils.findFirstModelEngine(user); + engineId = AutomationGenerationUtils.findFirstModelEngine(user); } if (engineId == null || engineId.trim().isEmpty()) { throw new IllegalArgumentException( @@ -182,14 +184,19 @@ public NounMetadata execute() { } } + // Wrap user content to prevent prompt injection + String safeDescription = "```user-request\n" + description.trim() + "\n```"; + StringBuilder initialMsg = new StringBuilder(); - initialMsg.append(AutomationExecutionUtils.buildAvailableEnginesSection(user)).append("\n"); + initialMsg.append(AutomationGenerationUtils.buildAvailableEnginesSection(user)).append("\n"); if (currentDoc != null) { classLogger.info("BuildAutomation edit mode: project={}, docLength={}", projectId, currentDoc.length()); - initialMsg.append("## Existing automation to modify\n").append(currentDoc).append("\n\n"); - initialMsg.append("## User modification request\n").append(description.trim()); + // Wrap currentDoc to prevent prompt injection + String safeCurrentDoc = "```automation-json\n" + currentDoc + "\n```"; + initialMsg.append("## Existing automation to modify\n").append(safeCurrentDoc).append("\n\n"); + initialMsg.append("## User modification request\n").append(safeDescription); } else { - initialMsg.append("## User request\n").append(description.trim()); + initialMsg.append("## User request\n").append(safeDescription); } // Fresh room per build request - isolated tool-call context, no history bleed @@ -198,7 +205,7 @@ public NounMetadata execute() { + pidClean.substring(0, Math.min(8, pidClean.length())) + Long.toString(System.currentTimeMillis(), 36); - Map options = AutomationExecutionUtils.buildEngineMcpOptions(user, SYSTEM_PROMPT); + Map options = AutomationGenerationUtils.buildEngineMcpOptions(user, SYSTEM_PROMPT); RoomUtils.createRoomIfNotExists(roomId, this.insight, modelEngine, initialMsg.toString(), null, options, null, projectId, null); @@ -231,8 +238,8 @@ public NounMetadata execute() { "The AI model did not return a response. Try again or start with a blank automation."); } - String docJson = AutomationExecutionUtils.stripCodeFences(finalText.trim()); - AutomationExecutionUtils.validateGeneratedDoc(docJson); + String docJson = AutomationGenerationUtils.stripCodeFences(finalText.trim()); + AutomationGenerationUtils.validateGeneratedDoc(docJson); classLogger.info("BuildAutomation finished: project={}", projectId); return new NounMetadata(docJson, PixelDataType.CONST_STRING, PixelOperationType.OPERATION); diff --git a/src/prerna/reactor/automation/CancelAutomationRunReactor.java b/src/prerna/reactor/automation/CancelAutomationRunReactor.java index 0d22c1f2988..49ca0c31309 100644 --- a/src/prerna/reactor/automation/CancelAutomationRunReactor.java +++ b/src/prerna/reactor/automation/CancelAutomationRunReactor.java @@ -60,7 +60,7 @@ public class CancelAutomationRunReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(CancelAutomationRunReactor.class); - // Not standardized in ReactorKeysEnum — matches the local-key convention used by + // Not standardized in ReactorKeysEnum - matches the local-key convention used by // prerna.reactor.agent (e.g. StopAgentRunReactor.RUN_ID_KEY). private static final String RUN_ID_KEY = "runId"; @@ -131,7 +131,7 @@ public String getReactorDescription() { @Override public Map getMcpToolMetadata() { Map meta = new HashMap<>(); - // Cancelling a run is a mutating, side-effecting action — requires explicit confirmation. + // Cancelling a run is a mutating, side-effecting action - requires explicit confirmation. meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); return meta; } diff --git a/src/prerna/reactor/automation/CreateAutomationReactor.java b/src/prerna/reactor/automation/CreateAutomationReactor.java index 69208d6504e..154007c5f86 100644 --- a/src/prerna/reactor/automation/CreateAutomationReactor.java +++ b/src/prerna/reactor/automation/CreateAutomationReactor.java @@ -27,6 +27,8 @@ *******************************************************************************/ package prerna.reactor.automation; +import prerna.reactor.automation.utils.PixelExecutionUtils; + import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; diff --git a/src/prerna/reactor/automation/ExplainAutomationReactor.java b/src/prerna/reactor/automation/ExplainAutomationReactor.java index 2eea7fa4d55..3f6152b6a66 100644 --- a/src/prerna/reactor/automation/ExplainAutomationReactor.java +++ b/src/prerna/reactor/automation/ExplainAutomationReactor.java @@ -27,10 +27,9 @@ *******************************************************************************/ package prerna.reactor.automation; -import java.io.File; -import java.io.IOException; +import prerna.reactor.automation.utils.AutomationGenerationUtils; + import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.util.HashMap; import java.util.Map; @@ -42,11 +41,11 @@ import prerna.auth.utils.SecurityProjectUtils; import prerna.engine.api.IModelEngine; import prerna.reactor.AbstractReactor; +import prerna.reactor.automation.utils.AutomationExecutionUtils; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; import prerna.sablecc2.om.ReactorKeysEnum; import prerna.sablecc2.om.nounmeta.NounMetadata; -import prerna.util.AssetUtility; import prerna.util.Utility; /** @@ -56,7 +55,7 @@ * *

Suitable for sharing with non-technical stakeholders or onboarding new team members. * - *

Uses the same model-engine resolution logic as {@link GenerateAutomationReactor}: + *

Uses the same model-engine resolution logic as {@link BuildAutomationReactor}: * uses the {@code engine} param if provided, otherwise falls back to the first accessible MODEL engine. * *

Pixel: {@code ExplainAutomation(project=["appId"])} @@ -73,7 +72,7 @@ public class ExplainAutomationReactor extends AbstractReactor { "You are a helpful assistant that explains software automation workflows to non-technical users. " + "Given the following automation JSON definition, write a 2-3 sentence plain-English explanation " + "of what this automation does when it runs. " - + "Write for a non-technical audience — do not mention JSON, nodes, config keys, or technical terms. " + + "Write for a non-technical audience - do not mention JSON, nodes, config keys, or technical terms. " + "Start your response with: 'When you run this automation, it will...'"; /** @@ -138,16 +137,16 @@ public NounMetadata execute() { classLogger.info("ExplainAutomationReactor: suggest mode (in-memory content), project={}", projectId); } else { classLogger.info("ExplainAutomationReactor: narrate mode (saved file), project={}", projectId); - doc = loadCurrentDoc(projectId); + doc = AutomationExecutionUtils.loadAutomationDocOrEmpty(projectId); if (doc.length() > 50_000) { doc = doc.substring(0, 50_000); } systemPrompt = NARRATION_SYSTEM_PROMPT; } - // Resolve model engine — provided ID or first accessible MODEL engine + // Resolve model engine - provided ID or first accessible MODEL engine if (engineId == null || engineId.trim().isEmpty()) { - engineId = AutomationExecutionUtils.findFirstModelEngine(user); + engineId = AutomationGenerationUtils.findFirstModelEngine(user); } if (engineId == null || engineId.trim().isEmpty()) { throw new IllegalArgumentException( @@ -179,7 +178,7 @@ public NounMetadata execute() { throw new RuntimeException("AI explanation failed: " + e.getMessage(), e); } - String explanation = AutomationExecutionUtils.extractResponseText(response); + String explanation = AutomationGenerationUtils.extractResponseText(response); if (explanation == null || explanation.isBlank()) { throw new IllegalStateException( "The AI model did not return an explanation. Try again."); @@ -189,19 +188,6 @@ public NounMetadata execute() { return new NounMetadata(explanation.trim(), PixelDataType.CONST_STRING, PixelOperationType.OPERATION); } - private String loadCurrentDoc(String projectId) { - try { - String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); - File automationFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); - if (automationFile.exists() && automationFile.isFile()) { - return Files.readString(automationFile.toPath(), StandardCharsets.UTF_8); - } - } catch (IOException e) { - classLogger.warn("Could not read automation.json for project {} — returning empty doc", projectId, e); - } - return "{\"version\":1,\"graph\":{\"nodes\":[],\"edges\":[]}}"; - } - @Override public String getReactorDescription() { return "Generates a plain-English explanation of what a saved automation does when run, " diff --git a/src/prerna/reactor/automation/GenerateAutomationReactor.java b/src/prerna/reactor/automation/GenerateAutomationReactor.java deleted file mode 100644 index 3a758acc9be..00000000000 --- a/src/prerna/reactor/automation/GenerateAutomationReactor.java +++ /dev/null @@ -1,407 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.reactor.automation; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.json.JSONArray; -import org.json.JSONObject; - -import prerna.auth.User; -import prerna.auth.utils.SecurityEngineUtils; -import prerna.auth.utils.SecurityProjectUtils; -import prerna.engine.api.IModelEngine; -import prerna.reactor.AbstractReactor; -import prerna.sablecc2.om.PixelDataType; -import prerna.sablecc2.om.PixelOperationType; -import prerna.sablecc2.om.ReactorKeysEnum; -import prerna.sablecc2.om.nounmeta.NounMetadata; -import prerna.util.Utility; - -/** - * Uses an LLM to scaffold a starter {@link AutomationDocument} from a plain-English description. - * - *

Pixel: {@code GenerateAutomation(project=["appId"], description=["what it should do"], engine=["modelEngineId"])} - * - *

Generation is two-pass: pass 1 builds the workflow structure and picks engines; pass 2 fetches - * the real DB schema for any chosen database engines and rewrites SQL expressions and model commands - * with accurate table/column names. Pass 2 is skipped when no database-engine nodes exist. - * - *

Returns the generated JSON as a string (same shape as {@code GetAutomation}). The caller is - * expected to display it for review and save it via {@code SaveAutomation} — this reactor does NOT - * persist anything. - */ -public class GenerateAutomationReactor extends AbstractReactor { - - private static final Logger classLogger = LogManager.getLogger(GenerateAutomationReactor.class); - - private static final int DESCRIPTION_MAX_CHARS = 1000; - private static final String PENDING_SQL = "PENDING_SQL_GENERATION"; - - /** - * Pass 1 system prompt: structural generation only. - * The LLM picks node types, engines, labels, and output vars. - * DB expressions are set to PENDING_SQL_GENERATION for pass 2 to fill in. - */ - private static final String SYSTEM_PROMPT_STRUCTURAL = """ -You are a workflow builder assistant. Generate an automation graph JSON document from the user's plain-English description. - -## Available node types -Each node has these fields: id (string), type (string), label (string), position ({x:0,y:0}), outputVar (string), config (object). - -Node types and their config shapes: -- trigger: config={"mode":"manual"} — always the first node, outputVar="trigger_out" -- database-engine: config={"engineId":"","operation":"query","expression":"PENDING_SQL_GENERATION","limit":50,"commit":false} — SQL queries, outputVar="db_out" -- model-engine: config={"engineId":"","operation":"llm","command":"","context":"","paramValues":"","values":"","image":"","prompt":"","entities":""} — LLM calls, outputVar="model_out" -- vector-engine: config={"engineId":"","operation":"search","command":"","limit":5,"filters":"","metaFilters":"","filePath":"","source":"","space":"","filePaths":"","paramValues":"","fileNames":""} — semantic search, outputVar="vector_out" -- storage-engine: config={"engineId":"","operation":"list","storagePath":"/","filePath":"","metadata":""} — file storage, outputVar="storage_out" -- function-engine: config={"engineId":"","operation":"execute","params":""} — custom functions, outputVar="fn_out" -- app: config={"pixel":"PENDING_PIXEL_EXPRESSION","appId":""} — run a custom reactor or arbitrary Pixel, outputVar="pixel_out" -- wait: config={"seconds":"5"} — pause between steps, outputVar="wait_out" - -## Variable substitution -Reference upstream node outputs in config fields using ${outputVar} (e.g. ${db_out}, ${model_out}). -NEVER use SQL parameterized syntax ($1, $2, ?, :param) — those are not supported and will cause runtime errors. -In SQL expressions, always wrap ${outputVar} in single quotes for string/UUID values: WHERE col = '${varName}'. NEVER use double quotes — PostgreSQL treats double-quoted values as column names. - -## Rules -1. Always start with a trigger node (id="trigger-1"). -2. Use only node types from the list above. -3. Set engineId from the available engines listed below. Leave empty string "" if no suitable engine exists. -4. Keep outputVar names unique and descriptive of what the node produces. -5. Make node labels action-oriented and specific to what the node does. -6. Build a realistic, useful graph — don't add unnecessary nodes. -7. For database-engine nodes: set expression to exactly "PENDING_SQL_GENERATION" — a second pass will fill in real SQL using the actual schema. -8. For app nodes: set pixel to exactly "PENDING_PIXEL_EXPRESSION" — a second pass will fill in the reactor call using the project's available reactors. -9. For model-engine nodes: "command" is the plain instruction to the LLM (e.g. "Summarize these cases and highlight urgent items"). Put the actual data by setting "context" to the upstream outputVar (e.g. "context":"${db_out}"). NEVER describe the data structure in prose inside "command" — the LLM will receive the real data at runtime via ${outputVar} substitution, not a description of it. -10. Respond with ONLY valid JSON. No markdown, no code fences, no explanation. - -## Response format -{"version":1,"description":"","graph":{"nodes":[...],"edges":[]}} -"""; - - /** - * Edit/iterate mode system prompt: same structure rules as pass 1, but instructs the LLM to - * treat the input as an existing document to modify rather than generate from scratch. - */ - private static final String SYSTEM_PROMPT_EDIT = """ -You are a workflow builder assistant. You are given an existing automation graph JSON document. Modify it based on the user's request. Preserve all steps and structure that are not directly affected by the request. You may add, remove, or change nodes as needed. Always keep a trigger node as the first node. - -## Available node types -Each node has these fields: id (string), type (string), label (string), position ({x:0,y:0}), outputVar (string), config (object). - -Node types and their config shapes: -- trigger: config={"mode":"manual"} — always the first node, outputVar="trigger_out" -- database-engine: config={"engineId":"","operation":"query","expression":"PENDING_SQL_GENERATION","limit":50,"commit":false} — SQL queries, outputVar="db_out" -- model-engine: config={"engineId":"","operation":"llm","command":"","context":"","paramValues":"","values":"","image":"","prompt":"","entities":""} — LLM calls, outputVar="model_out" -- vector-engine: config={"engineId":"","operation":"search","command":"","limit":5,"filters":"","metaFilters":"","filePath":"","source":"","space":"","filePaths":"","paramValues":"","fileNames":""} — semantic search, outputVar="vector_out" -- storage-engine: config={"engineId":"","operation":"list","storagePath":"/","filePath":"","metadata":""} — file storage, outputVar="storage_out" -- function-engine: config={"engineId":"","operation":"execute","params":""} — custom functions, outputVar="fn_out" -- app: config={"pixel":"PENDING_PIXEL_EXPRESSION","appId":""} — run a custom reactor or arbitrary Pixel, outputVar="pixel_out" -- wait: config={"seconds":"5"} — pause between steps, outputVar="wait_out" - -## Variable substitution -Reference upstream node outputs in config fields using ${outputVar} (e.g. ${db_out}, ${model_out}). -NEVER use SQL parameterized syntax ($1, $2, ?, :param) — those are not supported and will cause runtime errors. -In SQL expressions, always wrap ${outputVar} in single quotes for string/UUID values: WHERE col = '${varName}'. NEVER use double quotes — PostgreSQL treats double-quoted values as column names. - -## Rules -1. Always start with a trigger node (id="trigger-1"). -2. Use only node types from the list above. -3. Set engineId from the available engines listed below. Leave empty string "" if no suitable engine exists. -4. Keep outputVar names unique and descriptive of what the node produces. -5. Make node labels action-oriented and specific to what the node does. -6. Build a realistic, useful graph — don't add unnecessary nodes. -7. For database-engine nodes: set expression to exactly "PENDING_SQL_GENERATION" — a second pass will fill in real SQL using the actual schema. -8. For app nodes: set pixel to exactly "PENDING_PIXEL_EXPRESSION" — a second pass will fill in the reactor call using the project's available reactors. -9. For model-engine nodes: "command" is the plain instruction to the LLM (e.g. "Summarize these cases and highlight urgent items"). Put the actual data by setting "context" to the upstream outputVar (e.g. "context":"${db_out}"). NEVER describe the data structure in prose inside "command" — the LLM will receive the real data at runtime via ${outputVar} substitution, not a description of it. -10. Respond with ONLY valid JSON. No markdown, no code fences, no explanation. - -## Response format -{"version":1,"description":"","graph":{"nodes":[...],"edges":[]}} -"""; - - /** - * Pass 2 system prompt: schema-aware refinement. - * Rewrites PENDING_SQL_GENERATION, PENDING_PIXEL_EXPRESSION, and model commands using actual schema and reactor list. - */ - private static final String SYSTEM_PROMPT_REFINE = """ -You are given a workflow automation document along with context gathered after pass 1: the actual database schemas for chosen database engines, and the list of custom reactors available in this project. - -Your task: return a corrected copy of the document with these updates: -1. database-engine nodes: replace the "expression" value "PENDING_SQL_GENERATION" with a real SQL SELECT query using the actual table and column names from the schema. - - Write SQL that matches what the user asked for — use appropriate joins, filters, and columns based on the user's intent - - Do not add conditions or filters that aren't implied by the user's request - - Use column types to write type-safe SQL — don't compare VARCHAR columns to integers or assume a column's semantics from its name alone - - Use a reasonable LIMIT to avoid returning unbounded result sets - - NEVER use SQL parameterized placeholders ($1, $2, ?, :param) — the execution engine does not support bound parameters. For runtime values from upstream nodes, use ${outputVar} inline in the SQL string wrapped in single quotes (e.g. WHERE id = '${db_out}'). NEVER wrap ${outputVar} in double quotes — double quotes are SQL identifier delimiters and will cause a "column does not exist" error. For literal filters, hardcode the value directly. - - If no relevant table exists in the schema, set expression to "-- [replace with your SQL query]" and update the label to "Review: update this query" -2. model-engine nodes: ensure the "context" field contains the upstream outputVar reference (e.g. "${db_out}") so the actual data is passed to the LLM at runtime. The "command" must be a plain instruction only — NEVER describe the data structure or column names in prose inside "command". The LLM will see the real data via the context field; it does not need to be told what columns exist. -3. app nodes: replace the "pixel" value "PENDING_PIXEL_EXPRESSION" with a call to the most appropriate reactor from the available reactors list. - - Use the format: ReactorName(param="${varName}") referencing upstream outputVars where relevant - - If no reactor in the list clearly matches the intent, set pixel to "-- [describe the Pixel expression to write here]" - -Keep ALL other fields (ids, types, labels, outputVars, engineIds, edges, etc.) EXACTLY as they are. - -Respond with ONLY the complete updated JSON document. No markdown, no code fences, no explanation. -"""; - - /** Key for an optional base64-encoded current doc to modify (edit/iterate mode). */ - private static final String CURRENT_DOC_KEY = "currentDoc"; - - public GenerateAutomationReactor() { - this.keysToGet = new String[] { - ReactorKeysEnum.PROJECT.getKey(), - AutomationConstants.DOC_DESCRIPTION, - ReactorKeysEnum.ENGINE.getKey(), - CURRENT_DOC_KEY - }; - this.keyRequired = new int[] { 1, 1, 0, 0 }; - } - - @Override - public NounMetadata execute() { - organizeKeys(); - - User user = this.insight.getUser(); - if (user == null) { - throw new IllegalArgumentException("You are not properly logged in."); - } - - String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); - String description = this.keyValue.get(AutomationConstants.DOC_DESCRIPTION); - String engineId = this.keyValue.get(ReactorKeysEnum.ENGINE.getKey()); - - projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); - // View access is intentional — this reactor is read-only (no persistence). - // The caller saves the result via SaveAutomation, which enforces edit access. - if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { - throw new IllegalArgumentException("Project does not exist or user does not have access."); - } - - if (description == null || description.trim().isEmpty()) { - throw new IllegalArgumentException("A description of what the automation should do is required."); - } - // Decode base64-encoded description sent by the FE to prevent Pixel injection - try { - description = new String( - java.util.Base64.getDecoder().decode(description.trim()), - java.nio.charset.StandardCharsets.UTF_8); - } catch (IllegalArgumentException e) { - // Not base64-encoded (e.g. direct API call) — use the value as-is - } - if (description.length() > DESCRIPTION_MAX_CHARS) { - description = description.substring(0, DESCRIPTION_MAX_CHARS); - } - - // Resolve model engine — use provided ID or fall back to first available MODEL engine - if (engineId == null || engineId.trim().isEmpty()) { - engineId = AutomationExecutionUtils.findFirstModelEngine(user); - } - if (engineId == null || engineId.trim().isEmpty()) { - throw new IllegalArgumentException( - "No AI model engine is available. Add a model engine connection to generate an automation."); - } - if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { - throw new IllegalArgumentException( - "Model engine " + engineId + " does not exist or user does not have access."); - } - - IModelEngine modelEngine = Utility.getModel(engineId); - if (modelEngine == null) { - throw new IllegalArgumentException( - "Model engine " + engineId + " could not be loaded. It may no longer exist."); - } - - // Decode optional current doc (edit/iterate mode) - String currentDocRaw = this.keyValue.get(CURRENT_DOC_KEY); - String currentDoc = null; - if (currentDocRaw != null && !currentDocRaw.trim().isEmpty()) { - try { - currentDoc = new String( - java.util.Base64.getDecoder().decode(currentDocRaw.trim()), - java.nio.charset.StandardCharsets.UTF_8); - } catch (IllegalArgumentException e) { - // Not base64-encoded (e.g. direct API call) — use the value as-is - currentDoc = currentDocRaw; - } - if (currentDoc != null && currentDoc.length() > 50_000) { - currentDoc = currentDoc.substring(0, 50_000); - } - } - - Map paramMap = new HashMap<>(); - paramMap.put("use_history", false); - - // -- Pass 1: structural generation (or edit if current doc provided) ------------- - String enginesSection = AutomationExecutionUtils.buildAvailableEnginesSection(user); - String pass1Message; - String systemPrompt1; - if (currentDoc != null) { - classLogger.info("GenerateAutomation edit mode: project={}, docLength={}", projectId, currentDoc.length()); - systemPrompt1 = SYSTEM_PROMPT_EDIT; - pass1Message = enginesSection - + "\n## Existing automation\n```json\n" + currentDoc + "\n```" - + "\n## User's modification request\n" + description.trim(); - } else { - systemPrompt1 = SYSTEM_PROMPT_STRUCTURAL; - pass1Message = enginesSection + "\n## User's request\n" + description.trim(); - } - String raw = callLlm(modelEngine, systemPrompt1, pass1Message, paramMap, projectId); - raw = AutomationExecutionUtils.stripCodeFences(raw); - AutomationExecutionUtils.validateGeneratedDoc(raw); - - // -- Pass 2: schema-aware refinement (when DB nodes or app nodes are present) ---- - List dbEngineIds = extractDatabaseEngineIds(raw); - boolean hasAppNodes = extractHasAppNodes(raw); - if (!dbEngineIds.isEmpty() || hasAppNodes) { - classLogger.info("GenerateAutomation pass 2: db engines={}, appNodes={}, project={}", - dbEngineIds.size(), hasAppNodes, projectId); - StringBuilder pass2Message = new StringBuilder("## Workflow document from pass 1\n").append(raw).append("\n"); - if (!dbEngineIds.isEmpty()) { - pass2Message.append("\n").append(AutomationExecutionUtils.buildSchemaForEngineIds(dbEngineIds)); - } - if (hasAppNodes) { - pass2Message.append("\n").append(AutomationExecutionUtils.buildReactorListSection(projectId)); - } - try { - String refined = callLlm(modelEngine, SYSTEM_PROMPT_REFINE, pass2Message.toString(), paramMap, projectId); - refined = AutomationExecutionUtils.stripCodeFences(refined); - AutomationExecutionUtils.validateGeneratedDoc(refined); - raw = refined; - } catch (Exception e) { - classLogger.warn("GenerateAutomation pass 2 failed — returning pass 1 result. Reason: {}", e.getMessage()); - // Fall through: return pass 1 result unchanged - } - } - - return new NounMetadata(raw, PixelDataType.CONST_STRING, PixelOperationType.OPERATION); - } - - // -- Private helpers ------------------------------------------------------------- - - private String callLlm(IModelEngine modelEngine, String systemPrompt, String userMessage, - Map paramMap, String projectId) { - Map response; - try { - response = modelEngine.ask(systemPrompt + "\n\n" + userMessage, null, this.insight, paramMap).toMap(); - } catch (Exception e) { - classLogger.error("LLM call failed for GenerateAutomation on project {}", projectId, e); - throw new RuntimeException("AI generation failed: " + e.getMessage(), e); - } - String text = AutomationExecutionUtils.extractResponseText(response); - if (text == null || text.isBlank()) { - throw new IllegalStateException( - "The AI model did not return a response. Try again or start with a blank automation."); - } - return text; - } - - /** - * Parses a generated document and returns the engineIds of all database-engine nodes - * that have a non-blank engineId. Used to decide whether pass 2 is needed. - */ - private static List extractDatabaseEngineIds(String docJson) { - List ids = new ArrayList<>(); - try { - JSONObject doc = new JSONObject(docJson); - JSONObject graph = doc.optJSONObject("graph"); - if (graph == null) return ids; - JSONArray nodes = graph.optJSONArray("nodes"); - if (nodes == null) return ids; - for (int i = 0; i < nodes.length(); i++) { - JSONObject node = nodes.optJSONObject(i); - if (node == null) continue; - if ("database-engine".equals(node.optString("type"))) { - JSONObject config = node.optJSONObject("config"); - if (config != null) { - String dbId = config.optString("engineId", "").trim(); - if (!dbId.isEmpty() && !ids.contains(dbId)) { - ids.add(dbId); - } - } - } - } - } catch (Exception e) { - classLogger.warn("Failed to extract database engine IDs from generated doc", e); - } - return ids; - } - - /** - * Returns true if the document contains any app-type nodes whose pixel is still PENDING_PIXEL_EXPRESSION. - * Used to decide whether pass 2 needs to include the reactor list. - */ - private static boolean extractHasAppNodes(String docJson) { - try { - JSONObject doc = new JSONObject(docJson); - JSONObject graph = doc.optJSONObject("graph"); - if (graph == null) return false; - JSONArray nodes = graph.optJSONArray("nodes"); - if (nodes == null) return false; - for (int i = 0; i < nodes.length(); i++) { - JSONObject node = nodes.optJSONObject(i); - if (node == null) continue; - if ("app".equals(node.optString("type"))) { - JSONObject config = node.optJSONObject("config"); - if (config != null && "PENDING_PIXEL_EXPRESSION".equals(config.optString("pixel"))) { - return true; - } - } - } - } catch (Exception e) { - classLogger.warn("Failed to check for app nodes in generated doc", e); - } - return false; - } - - @Override - public String getReactorDescription() { - return "Uses an AI model to scaffold a starter automation graph from a plain-English description. " - + "Returns the generated document JSON — the caller must save it via SaveAutomation."; - } - - @Override - protected String getDescriptionForKey(String key) { - if (AutomationConstants.DOC_DESCRIPTION.equals(key)) { - return "Plain-English description of what the automation should do (max 1000 characters)."; - } else if (ReactorKeysEnum.PROJECT.getKey().equals(key)) { - return "Project ID that will own this automation."; - } else if (ReactorKeysEnum.ENGINE.getKey().equals(key)) { - return "Optional model engine ID to use for generation. Defaults to the first available MODEL engine."; - } else if (CURRENT_DOC_KEY.equals(key)) { - return "Optional base64-encoded JSON of an existing automation document. When provided, the LLM modifies the existing document rather than generating from scratch."; - } - return super.getDescriptionForKey(key); - } -} diff --git a/src/prerna/reactor/automation/GenerateNodeLabelReactor.java b/src/prerna/reactor/automation/GenerateNodeLabelReactor.java index 101b17cb36e..3fe6d345dad 100644 --- a/src/prerna/reactor/automation/GenerateNodeLabelReactor.java +++ b/src/prerna/reactor/automation/GenerateNodeLabelReactor.java @@ -27,6 +27,10 @@ *******************************************************************************/ package prerna.reactor.automation; +import prerna.reactor.automation.utils.AutomationGenerationUtils; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + import java.util.Base64; import java.util.HashMap; import java.util.Map; @@ -101,7 +105,7 @@ public NounMetadata execute() { configJson = configEncoded; } - String engineId = AutomationExecutionUtils.findFirstModelEngine(user); + String engineId = AutomationGenerationUtils.findFirstModelEngine(user); if (engineId == null || engineId.isBlank()) { throw new IllegalArgumentException( "No AI model engine is available. Add a model engine connection to use this feature."); @@ -132,7 +136,7 @@ public NounMetadata execute() { throw new RuntimeException("Label generation failed: " + e.getMessage(), e); } - String label = AutomationExecutionUtils.extractResponseText(response); + String label = AutomationGenerationUtils.extractResponseText(response); if (label == null || label.isBlank()) { throw new IllegalStateException("The AI model did not return a label. Try again."); } diff --git a/src/prerna/reactor/automation/GenerateRunSummaryReactor.java b/src/prerna/reactor/automation/GenerateRunSummaryReactor.java index acf88e96afd..8765dade622 100644 --- a/src/prerna/reactor/automation/GenerateRunSummaryReactor.java +++ b/src/prerna/reactor/automation/GenerateRunSummaryReactor.java @@ -27,6 +27,10 @@ *******************************************************************************/ package prerna.reactor.automation; +import prerna.reactor.automation.utils.AutomationGenerationUtils; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -104,7 +108,7 @@ public NounMetadata execute() { classLogger.warn("GenerateRunSummary: no node outputs found for run {}", runId); } - String engineId = AutomationExecutionUtils.findFirstModelEngine(user); + String engineId = AutomationGenerationUtils.findFirstModelEngine(user); if (engineId == null || engineId.isBlank()) { throw new IllegalArgumentException( "No AI model engine is available. Add a model engine connection to use this feature."); @@ -133,7 +137,7 @@ public NounMetadata execute() { throw new RuntimeException("Summary generation failed: " + e.getMessage(), e); } - String summary = AutomationExecutionUtils.extractResponseText(response); + String summary = AutomationGenerationUtils.extractResponseText(response); if (summary == null || summary.isBlank()) { throw new IllegalStateException("The AI model did not return a summary. Try again."); } diff --git a/src/prerna/reactor/automation/GetAutomationConfigReactor.java b/src/prerna/reactor/automation/GetAutomationConfigReactor.java index ed2f99cae1d..5f1f4d55ee6 100644 --- a/src/prerna/reactor/automation/GetAutomationConfigReactor.java +++ b/src/prerna/reactor/automation/GetAutomationConfigReactor.java @@ -27,10 +27,13 @@ *******************************************************************************/ package prerna.reactor.automation; +import prerna.reactor.automation.utils.AutomationExecutionUtils; + import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -42,6 +45,7 @@ import prerna.auth.utils.SecurityProjectUtils; import prerna.reactor.AbstractReactor; +import prerna.util.Utility; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; import prerna.sablecc2.om.ReactorKeysEnum; @@ -50,13 +54,13 @@ /** * Returns the automation environment config ({@code automation_config.json}) for a project. - * Sensitive values are masked in the response — only the key and a placeholder are returned. + * Sensitive values are masked in the response - only the key and a placeholder are returned. * - *

There are three "get" reactors — each reads something different: + *

There are three "get" reactors - each reads something different: *

    - *
  • {@link GetAutomationReactor GetAutomation} — reads {@code automation.json}: the pipeline graph
    • - *
  • {@code GetAutomationConfig} (this reactor) — reads {@code automation_config.json}: key/value env vars and secrets
    • - *
  • {@link GetAutomationRunReactor GetAutomationRun} — reads live run state from the DB
    • + *
  • {@link GetAutomationReactor GetAutomation} - reads {@code automation.json}: the pipeline graph
    • + *
  • {@code GetAutomationConfig} (this reactor) - reads {@code automation_config.json}: key/value env vars and secrets
    • + *
  • {@link GetAutomationRunReactor GetAutomationRun} - reads live run state from the DB
    • *
* *

Pixel: {@code GetAutomationConfig(project=["appId"])} @@ -67,6 +71,7 @@ public class GetAutomationConfigReactor extends AbstractReactor { public GetAutomationConfigReactor() { this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey() }; + this.keyRequired = new int[] { 1 }; } @Override @@ -84,9 +89,13 @@ public NounMetadata execute() { } String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); - File configFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_CONFIG_FILE_NAME); + File configFile = Paths.get(portalsFolder, AutomationConstants.AUTOMATION_CONFIG_FILE_NAME).toFile(); + String normalizedPath = Utility.normalizePath(configFile.getAbsolutePath()); + if (!normalizedPath.startsWith(portalsFolder)) { + throw new IllegalArgumentException("Invalid file path"); + } - if (!configFile.exists()) { + if (!configFile.exists() || !configFile.isFile()) { return new NounMetadata(new ArrayList<>(), PixelDataType.VECTOR, PixelOperationType.OPERATION); } diff --git a/src/prerna/reactor/automation/GetAutomationReactor.java b/src/prerna/reactor/automation/GetAutomationReactor.java index ffef6b63100..6b383912ef5 100644 --- a/src/prerna/reactor/automation/GetAutomationReactor.java +++ b/src/prerna/reactor/automation/GetAutomationReactor.java @@ -27,10 +27,13 @@ *******************************************************************************/ package prerna.reactor.automation; +import prerna.reactor.automation.utils.AutomationExecutionUtils; + import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -52,13 +55,13 @@ * Returns the automation definition (the saved {@code automation.json} graph) for a project. * Returns an empty graph document when no automation has been saved yet. * - *

There are three "get" reactors — each reads something different: + *

There are three "get" reactors - each reads something different: *

    - *
  • {@code GetAutomation} (this reactor) — reads {@code automation.json}: the pipeline graph + *
  • {@code GetAutomation} (this reactor) - reads {@code automation.json}: the pipeline graph * (nodes, edges, output transforms). Static config written by {@link SaveAutomationReactor}.
    • - *
  • {@link GetAutomationConfigReactor GetAutomationConfig} — reads {@code automation_config.json}: + *
  • {@link GetAutomationConfigReactor GetAutomationConfig} - reads {@code automation_config.json}: * key/value env vars and secrets; sensitive values are masked in the response.
    • - *
  • {@link GetAutomationRunReactor GetAutomationRun} — reads live run state from the DB + *
  • {@link GetAutomationRunReactor GetAutomationRun} - reads live run state from the DB * (AUTOMATION_RUNS + AUTOMATION_NODE_OUTPUTS); used by the FE to poll execution progress.
    • *
* @@ -93,7 +96,11 @@ public NounMetadata execute() { } String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); - File automationFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); + File automationFile = Paths.get(portalsFolder, AutomationConstants.AUTOMATION_FILE_NAME).toFile(); + String normalizedPath = Utility.normalizePath(automationFile.getAbsolutePath()); + if (!normalizedPath.startsWith(portalsFolder)) { + throw new IllegalArgumentException("Invalid file path"); + } if (!automationFile.exists() || !automationFile.isFile()) { // return empty graph document for brand-new automations diff --git a/src/prerna/reactor/automation/GetAutomationRunReactor.java b/src/prerna/reactor/automation/GetAutomationRunReactor.java index 32695ee7997..939c1f77c4e 100644 --- a/src/prerna/reactor/automation/GetAutomationRunReactor.java +++ b/src/prerna/reactor/automation/GetAutomationRunReactor.java @@ -32,6 +32,9 @@ import java.util.List; import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import prerna.auth.utils.SecurityProjectUtils; import prerna.reactor.AbstractReactor; import prerna.sablecc2.om.PixelDataType; @@ -48,13 +51,15 @@ */ public class GetAutomationRunReactor extends AbstractReactor { + private static final Logger classLogger = LogManager.getLogger(GetAutomationRunReactor.class); + // Not standardized in ReactorKeysEnum — matches the local-key convention used by // prerna.reactor.agent (e.g. GetAgentRunReactor.RUN_ID_KEY). private static final String RUN_ID_KEY = "runId"; public GetAutomationRunReactor() { - this.keysToGet = new String[]{ ReactorKeysEnum.PROJECT.getKey(), RUN_ID_KEY }; - this.keyRequired = new int[]{ 1, 1 }; + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), RUN_ID_KEY }; + this.keyRequired = new int[] { 1, 1 }; } @Override @@ -86,22 +91,7 @@ public NounMetadata execute() { } List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(runId); - List> nodeResults = new ArrayList<>(); - - for (Map nodeOutput : nodeOutputs) { - Map nodeResult = new HashMap<>(); - nodeResult.put(AutomationConstants.NODE_ID, nodeOutput.get(AutomationConstants.NODE_ID)); - nodeResult.put(AutomationConstants.NODE_LABEL, nodeOutput.get(AutomationConstants.NODE_LABEL)); - nodeResult.put(AutomationConstants.STATUS, nodeOutput.get(AutomationConstants.STATUS)); - nodeResult.put(AutomationConstants.DURATION_MS, nodeOutput.get(AutomationConstants.DURATION_MS)); - String outputForDisplay = (String) nodeOutput.get(AutomationConstants.OUTPUT_VALUE); - if (outputForDisplay == null || outputForDisplay.isBlank()) { - outputForDisplay = (String) nodeOutput.get(AutomationConstants.OUTPUT_PREVIEW); - } - nodeResult.put(AutomationConstants.OUTPUT_PREVIEW, outputForDisplay); - nodeResult.put(AutomationConstants.ERROR_MESSAGE, nodeOutput.get(AutomationConstants.ERROR_MESSAGE)); - nodeResults.add(nodeResult); - } + List> nodeResults = AutomationDatabaseUtility.buildNodeResults(nodeOutputs); runDetail.put(AutomationConstants.RESULT_NODE_RESULTS, nodeResults); return new NounMetadata(runDetail, PixelDataType.MAP, PixelOperationType.OPERATION); diff --git a/src/prerna/reactor/automation/GetAutomationSchemaReactor.java b/src/prerna/reactor/automation/GetAutomationSchemaReactor.java index fc65f963570..16561d0cc5f 100644 --- a/src/prerna/reactor/automation/GetAutomationSchemaReactor.java +++ b/src/prerna/reactor/automation/GetAutomationSchemaReactor.java @@ -27,6 +27,8 @@ *******************************************************************************/ package prerna.reactor.automation; +import prerna.reactor.automation.utils.AutomationExecutionUtils; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; diff --git a/src/prerna/reactor/automation/GetAutomationStructureReactor.java b/src/prerna/reactor/automation/GetAutomationStructureReactor.java index 8821f56052a..c83c3f8ca4d 100644 --- a/src/prerna/reactor/automation/GetAutomationStructureReactor.java +++ b/src/prerna/reactor/automation/GetAutomationStructureReactor.java @@ -28,21 +28,16 @@ package prerna.reactor.automation; import java.io.File; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.google.gson.reflect.TypeToken; - import prerna.auth.utils.SecurityProjectUtils; import prerna.reactor.AbstractReactor; +import prerna.reactor.automation.utils.AutomationExecutionUtils; +import prerna.util.Utility; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; import prerna.sablecc2.om.ReactorKeysEnum; @@ -60,8 +55,6 @@ */ public class GetAutomationStructureReactor extends AbstractReactor { - private static final Logger classLogger = LogManager.getLogger(GetAutomationStructureReactor.class); - private static final String RESULT_KEY_DESCRIPTION = "description"; private static final String RESULT_KEY_NODES = "nodes"; @@ -89,30 +82,27 @@ public NounMetadata execute() { result.put(RESULT_KEY_NODES, new ArrayList<>()); String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); - File automationFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); + File automationFile = Paths.get(portalsFolder, AutomationConstants.AUTOMATION_FILE_NAME).toFile(); + String normalizedPath = Utility.normalizePath(automationFile.getAbsolutePath()); + if (!normalizedPath.startsWith(portalsFolder)) { + throw new IllegalArgumentException("Invalid file path"); + } if (!automationFile.exists() || !automationFile.isFile()) { return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); } - try { - String json = Files.readString(automationFile.toPath(), StandardCharsets.UTF_8); - Map doc = AutomationExecutionUtils.GSON.fromJson(json, - new TypeToken>() {}.getType()); + Map doc = AutomationExecutionUtils.loadAutomationDoc(projectId); - String description = (String) doc.getOrDefault(AutomationConstants.DOC_DESCRIPTION, ""); - result.put(RESULT_KEY_DESCRIPTION, description != null ? description : ""); + String description = (String) doc.getOrDefault(AutomationConstants.DOC_DESCRIPTION, ""); + result.put(RESULT_KEY_DESCRIPTION, description != null ? description : ""); - Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); - if (graph != null) { - List nodes = (List) graph.get(AutomationConstants.DOC_NODES); - if (nodes != null) { - result.put(RESULT_KEY_NODES, buildNodeSummary(nodes)); - } + Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); + if (graph != null) { + List nodes = (List) graph.get(AutomationConstants.DOC_NODES); + if (nodes != null) { + result.put(RESULT_KEY_NODES, buildNodeSummary(nodes)); } - } catch (IOException e) { - classLogger.error("Error reading automation.json for project {}", projectId, e); - throw new IllegalArgumentException("Unable to read automation: " + e.getMessage()); } return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); diff --git a/src/prerna/reactor/automation/GetReactorSignatureReactor.java b/src/prerna/reactor/automation/GetReactorSignatureReactor.java index 667aee797ae..560ee198ec7 100644 --- a/src/prerna/reactor/automation/GetReactorSignatureReactor.java +++ b/src/prerna/reactor/automation/GetReactorSignatureReactor.java @@ -53,10 +53,10 @@ * *

Returns a JSON object with: *

    - *
  • {@code template} — a filled Pixel call showing each param placeholder, e.g. + *
  • {@code template} - a filled Pixel call showing each param placeholder, e.g. * {@code MyReactor(requiredKey=[""], optionalKey="")}
    • - *
  • {@code description} — the reactor's one-line description, or empty string if none
    • - *
  • {@code hasParams} — boolean, false when the reactor declares no keys
    • + *
  • {@code description} - the reactor's one-line description, or empty string if none
    • + *
  • {@code hasParams} - boolean, false when the reactor declares no keys
    • *
* *

On any failure (project not found, reactor not found, bad metadata) returns a minimal @@ -119,7 +119,7 @@ private static JSONObject buildSignature(String projectId, String reactorName) { return fallback(result, reactorName); } - // Description — best-effort + // Description - best-effort String description = ""; try { String d = reactor.getReactorDescription(); @@ -196,7 +196,7 @@ private static JSONObject buildParamMeta(String key, JSONObject prop, boolean re String type = prop.optString("type", "string"); meta.put("type", type); String desc = prop.optString("description", ""); - // Suppress the default placeholder description — it adds no value + // Suppress the default placeholder description - it adds no value if (!desc.isBlank() && !desc.equals("No description present")) { meta.put("description", desc); } diff --git a/src/prerna/reactor/automation/ListAutomationRunsReactor.java b/src/prerna/reactor/automation/ListAutomationRunsReactor.java index c4ad3ef4676..c56c017aa61 100644 --- a/src/prerna/reactor/automation/ListAutomationRunsReactor.java +++ b/src/prerna/reactor/automation/ListAutomationRunsReactor.java @@ -31,6 +31,9 @@ import java.util.List; import java.util.Map; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import prerna.auth.utils.SecurityProjectUtils; import prerna.reactor.AbstractReactor; import prerna.sablecc2.om.PixelDataType; @@ -47,18 +50,24 @@ */ public class ListAutomationRunsReactor extends AbstractReactor { + private static final Logger classLogger = LogManager.getLogger(ListAutomationRunsReactor.class); + public ListAutomationRunsReactor() { - this.keysToGet = new String[]{ ReactorKeysEnum.PROJECT.getKey(), ReactorKeysEnum.LIMIT.getKey() }; - this.keyRequired = new int[]{ 1, 0 }; + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), ReactorKeysEnum.LIMIT.getKey() }; + this.keyRequired = new int[] { 1, 0 }; } @Override public NounMetadata execute() { organizeKeys(); - String projectId = this.keyValue.get(this.keysToGet[0]); - String limitStr = this.keyValue.get(this.keysToGet[1]); + String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); + String limitStr = this.keyValue.get(ReactorKeysEnum.LIMIT.getKey()); int limit = parseLimit(limitStr); + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { throw new IllegalArgumentException("Project does not exist or user does not have access"); diff --git a/src/prerna/reactor/automation/QuickEditAutomationReactor.java b/src/prerna/reactor/automation/QuickEditAutomationReactor.java index 6a2b4a040d7..fc5ccfc952e 100644 --- a/src/prerna/reactor/automation/QuickEditAutomationReactor.java +++ b/src/prerna/reactor/automation/QuickEditAutomationReactor.java @@ -27,10 +27,9 @@ *******************************************************************************/ package prerna.reactor.automation; -import java.io.File; -import java.io.IOException; +import prerna.reactor.automation.utils.PixelExecutionUtils; + import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.util.Base64; import java.util.HashMap; import java.util.LinkedHashMap; @@ -42,16 +41,17 @@ import prerna.auth.utils.SecurityProjectUtils; import prerna.reactor.AbstractReactor; +import prerna.reactor.automation.utils.AutomationExecutionUtils; import prerna.reactor.agent.mcp.MCPUtility; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; import prerna.sablecc2.om.ReactorKeysEnum; import prerna.sablecc2.om.nounmeta.NounMetadata; -import prerna.util.AssetUtility; + /** * Headless automation editor. The LLM provides a plain-English description of the desired change; - * this reactor chains {@code GenerateAutomation} (edit mode) and {@code SaveAutomation} silently + * this reactor chains {@code BuildAutomation} (edit mode) and {@code SaveAutomation} silently * without opening any UI. Returns a compact summary so the LLM can narrate the result. * *

Use {@code EditAutomation} instead when the change is complex or when the user wants to @@ -59,203 +59,185 @@ * *

Threading: {@link PixelExecutionUtils#runAndCollect} blocks the caller thread for the full * LLM round-trip (up to {@link AutomationConstants#DEFAULT_TIMEOUT_SECONDS} seconds). Acceptable - * for MVP usage volumes — revisit under concurrent load. + * for MVP usage volumes - revisit under concurrent load. * *

Pixel: {@code QuickEditAutomation(project=["appId"], editDescription=["change the SQL to last 7 days"])} */ public class QuickEditAutomationReactor extends AbstractReactor { - private static final Logger classLogger = LogManager.getLogger(QuickEditAutomationReactor.class); - - private static final String EDIT_DESCRIPTION_KEY = "editDescription"; - - private static final String RESULT_SUCCESS = "success"; - private static final String RESULT_NODE_COUNT = "nodeCount"; - private static final String RESULT_MESSAGE = "message"; - - public QuickEditAutomationReactor() { - this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), EDIT_DESCRIPTION_KEY }; - this.keyRequired = new int[] { 1, 1 }; - } - - @Override - public NounMetadata execute() { - organizeKeys(); - String projectId = this.keyValue.get(this.keysToGet[0]); - String editDescription = this.keyValue.get(EDIT_DESCRIPTION_KEY); - - if (projectId == null || projectId.isEmpty()) { - throw new IllegalArgumentException("Must provide a project id"); - } - if (editDescription == null || editDescription.trim().isEmpty()) { - throw new IllegalArgumentException("Must provide an editDescription"); - } - - projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); - if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { - throw new IllegalArgumentException("Project does not exist or user does not have edit access"); - } - - classLogger.info("QuickEditAutomationReactor: starting edit for project {}", projectId); - - String currentDocJson = loadCurrentDoc(projectId); - - // GenerateAutomation decodes base64 to prevent Pixel injection — same pattern used by the FE - String encodedDesc = Base64.getEncoder() - .encodeToString(editDescription.trim().getBytes(StandardCharsets.UTF_8)); - String encodedDoc = Base64.getEncoder() - .encodeToString(currentDocJson.getBytes(StandardCharsets.UTF_8)); - - String generatePixel = String.format( - "GenerateAutomation(project=[\"%s\"], description=[\"%s\"], currentDoc=[\"%s\"]);", - projectId, encodedDesc, encodedDoc); - - classLogger.info("QuickEditAutomationReactor: calling GenerateAutomation for project {}", projectId); - Object raw; - try { - raw = PixelExecutionUtils.runAndCollect(this.insight, generatePixel); - } catch (PixelExecutionUtils.AutomationPixelException e) { - classLogger.error("GenerateAutomation pixel error for project {}", projectId, e); - throw new IllegalArgumentException("AI generation failed: " + e.getMessage()); - } - - if (raw == null) { - throw new IllegalArgumentException("GenerateAutomation returned no result for project: " + projectId); - } - String generatedJson = raw instanceof String ? (String) raw : raw.toString(); - if (generatedJson.isBlank()) { - throw new IllegalArgumentException("GenerateAutomation returned an empty result for project: " + projectId); - } - - Map generatedDoc = parseAndValidate(generatedJson, projectId); - - String encodedSave = Base64.getEncoder() - .encodeToString(generatedJson.getBytes(StandardCharsets.UTF_8)); - String savePixel = String.format( - "SaveAutomation(project=[\"%s\"], json=[\"%s\"]);", - projectId, encodedSave); - - classLogger.info("QuickEditAutomationReactor: calling SaveAutomation for project {}", projectId); - try { - PixelExecutionUtils.runAndCollect(this.insight, savePixel); - } catch (PixelExecutionUtils.AutomationPixelException e) { - classLogger.error("SaveAutomation pixel error for project {}", projectId, e); - throw new IllegalArgumentException("Failed to save automation: " + e.getMessage()); - } - - Map summary = buildSummary(generatedDoc); - classLogger.info("QuickEditAutomationReactor: completed for project {}", projectId); - return new NounMetadata(summary, PixelDataType.MAP, PixelOperationType.OPERATION); - } - - /** - * Reads the current automation.json from disk. - * Returns a minimal blank document if none exists — GenerateAutomation treats the absence of - * meaningful nodes as a fresh-start signal. - */ - private String loadCurrentDoc(String projectId) { - try { - String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); - File automationFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); - if (automationFile.exists() && automationFile.isFile()) { - return Files.readString(automationFile.toPath(), StandardCharsets.UTF_8); - } - } catch (IOException e) { - classLogger.warn("Could not read current automation.json for project {} — treating as blank", projectId, e); - } - return "{\"version\":1,\"graph\":{\"nodes\":[],\"edges\":[]}}"; - } - - /** - * Parses the generated JSON and validates its structure before allowing a save. - * Throws if the document is malformed or missing the trigger node — prevents corrupt saves. - */ - @SuppressWarnings("unchecked") - private Map parseAndValidate(String json, String projectId) { - Map doc; - try { - doc = AutomationExecutionUtils.GSON.fromJson(json, AutomationExecutionUtils.MAP_TYPE); - } catch (Exception e) { - classLogger.error("GenerateAutomation returned invalid JSON for project {}", projectId, e); - throw new IllegalArgumentException("AI generation returned invalid JSON — not saved."); - } - - Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); - if (graph == null) { - classLogger.error("GenerateAutomation result missing 'graph' field for project {}", projectId); - throw new IllegalArgumentException("AI generation result is missing 'graph' field — not saved."); - } - - List nodes = (List) graph.get(AutomationConstants.DOC_NODES); - if (nodes == null || nodes.isEmpty()) { - classLogger.error("GenerateAutomation result has no nodes for project {}", projectId); - throw new IllegalArgumentException("AI generation result has no nodes — not saved."); - } - - boolean hasTrigger = nodes.stream() - .filter(n -> n instanceof Map) - .map(n -> (Map) n) - .anyMatch(n -> AutomationConstants.NODE_TRIGGER.equals(n.get(AutomationConstants.NODE_FIELD_TYPE))); - - if (!hasTrigger) { - classLogger.error("GenerateAutomation result has no trigger node for project {}", projectId); - throw new IllegalArgumentException("AI generation result has no trigger node — not saved."); - } - - return doc; - } - - /** - * Builds the summary Map returned to the LLM — concise enough to fit in context without - * sending the full automation JSON. - */ - @SuppressWarnings("unchecked") - private static Map buildSummary(Map doc) { - Map summary = new LinkedHashMap<>(); - summary.put(RESULT_SUCCESS, true); - - try { - String desc = (String) doc.getOrDefault(AutomationConstants.DOC_DESCRIPTION, ""); - Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); - List nodes = graph != null ? (List) graph.get(AutomationConstants.DOC_NODES) : null; - int nodeCount = nodes != null ? nodes.size() : 0; - summary.put(RESULT_NODE_COUNT, nodeCount); - - String message = (desc != null && !desc.isBlank()) - ? "Automation updated successfully. Description: \"" + desc.trim() + "\". " + nodeCount + " steps." - : "Automation updated successfully. " + nodeCount + " steps."; - summary.put(RESULT_MESSAGE, message); - } catch (Exception e) { - classLogger.warn("Could not build summary from generated doc", e); - summary.put(RESULT_MESSAGE, "Automation updated successfully."); - } - - return summary; - } - - @Override - public Map getMcpToolMetadata() { - Map meta = new HashMap<>(); - meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); - return meta; - } - - @Override - public String getReactorDescription() { - return "Modifies an automation using AI generation without opening any UI. " - + "Chains GenerateAutomation (edit mode) and SaveAutomation silently. " - + "Use EditAutomation instead when the user needs to review the change before saving."; - } - - @Override - protected String getDescriptionForKey(String key) { - if (ReactorKeysEnum.PROJECT.getKey().equals(key)) { - return "The project ID of the automation to edit."; - } else if (EDIT_DESCRIPTION_KEY.equals(key)) { - return "Plain-language description of the change to make. " - + "Example: 'Change the SQL filter to pull records from the last 7 days'. " - + "Call GetAutomationStructure first to understand the current automation structure."; - } - return super.getDescriptionForKey(key); - } + private static final Logger classLogger = LogManager.getLogger(QuickEditAutomationReactor.class); + + private static final String EDIT_DESCRIPTION_KEY = "editDescription"; + + private static final String RESULT_SUCCESS = "success"; + private static final String RESULT_NODE_COUNT = "nodeCount"; + private static final String RESULT_MESSAGE = "message"; + + public QuickEditAutomationReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), EDIT_DESCRIPTION_KEY }; + this.keyRequired = new int[] { 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); + String editDescription = this.keyValue.get(EDIT_DESCRIPTION_KEY); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + if (editDescription == null || editDescription.trim().isEmpty()) { + throw new IllegalArgumentException("Must provide an editDescription"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have edit access"); + } + + classLogger.info("QuickEditAutomationReactor: starting edit for project {}", projectId); + + String currentDocJson = AutomationExecutionUtils.loadAutomationDocOrEmpty(projectId); + + // BuildAutomation decodes base64 to prevent Pixel injection - same pattern used by the FE + String encodedDesc = Base64.getEncoder() + .encodeToString(editDescription.trim().getBytes(StandardCharsets.UTF_8)); + String encodedDoc = Base64.getEncoder() + .encodeToString(currentDocJson.getBytes(StandardCharsets.UTF_8)); + + String generatePixel = String.format( + "BuildAutomation(project=[\"%s\"], description=[\"%s\"], currentDoc=[\"%s\"]);", + projectId, encodedDesc, encodedDoc); + + classLogger.info("QuickEditAutomationReactor: calling BuildAutomation for project {}", projectId); + Object raw; + try { + raw = PixelExecutionUtils.runAndCollect(this.insight, generatePixel); + } catch (PixelExecutionUtils.AutomationPixelException e) { + classLogger.error("BuildAutomation pixel error for project {}", projectId, e); + throw new IllegalArgumentException("AI generation failed: " + e.getMessage()); + } + + if (raw == null) { + throw new IllegalArgumentException("BuildAutomation returned no result for project: " + projectId); + } + String generatedJson = raw instanceof String ? (String) raw : raw.toString(); + if (generatedJson.isBlank()) { + throw new IllegalArgumentException("BuildAutomation returned an empty result for project: " + projectId); + } + + Map generatedDoc = parseAndValidate(generatedJson, projectId); + + String encodedSave = Base64.getEncoder() + .encodeToString(generatedJson.getBytes(StandardCharsets.UTF_8)); + String savePixel = String.format( + "SaveAutomation(project=[\"%s\"], json=[\"%s\"]);", + projectId, encodedSave); + + classLogger.info("QuickEditAutomationReactor: calling SaveAutomation for project {}", projectId); + try { + PixelExecutionUtils.runAndCollect(this.insight, savePixel); + } catch (PixelExecutionUtils.AutomationPixelException e) { + classLogger.error("SaveAutomation pixel error for project {}", projectId, e); + throw new IllegalArgumentException("Failed to save automation: " + e.getMessage()); + } + + Map summary = buildSummary(generatedDoc); + classLogger.info("QuickEditAutomationReactor: completed for project {}", projectId); + return new NounMetadata(summary, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + /** + * Parses the generated JSON and validates its structure before allowing a save. + * Throws if the document is malformed or missing the trigger node - prevents corrupt saves. + */ + @SuppressWarnings("unchecked") + private Map parseAndValidate(String json, String projectId) { + Map doc; + try { + doc = AutomationExecutionUtils.GSON.fromJson(json, AutomationExecutionUtils.MAP_TYPE); + } catch (Exception e) { + classLogger.error("BuildAutomation returned invalid JSON for project {}", projectId, e); + throw new IllegalArgumentException("AI generation returned invalid JSON - not saved."); + } + + Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); + if (graph == null) { + classLogger.error("BuildAutomation result missing 'graph' field for project {}", projectId); + throw new IllegalArgumentException("AI generation result is missing 'graph' field - not saved."); + } + + List nodes = (List) graph.get(AutomationConstants.DOC_NODES); + if (nodes == null || nodes.isEmpty()) { + classLogger.error("BuildAutomation result has no nodes for project {}", projectId); + throw new IllegalArgumentException("AI generation result has no nodes - not saved."); + } + + boolean hasTrigger = nodes.stream() + .filter(n -> n instanceof Map) + .map(n -> (Map) n) + .anyMatch(n -> AutomationConstants.NODE_TRIGGER.equals(n.get(AutomationConstants.NODE_FIELD_TYPE))); + + if (!hasTrigger) { + classLogger.error("BuildAutomation result has no trigger node for project {}", projectId); + throw new IllegalArgumentException("AI generation result has no trigger node - not saved."); + } + + return doc; + } + + /** + * Builds the summary Map returned to the LLM - concise enough to fit in context without + * sending the full automation JSON. + */ + @SuppressWarnings("unchecked") + private static Map buildSummary(Map doc) { + Map summary = new LinkedHashMap<>(); + summary.put(RESULT_SUCCESS, true); + + try { + String desc = (String) doc.getOrDefault(AutomationConstants.DOC_DESCRIPTION, ""); + Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); + List nodes = graph != null ? (List) graph.get(AutomationConstants.DOC_NODES) : null; + int nodeCount = nodes != null ? nodes.size() : 0; + summary.put(RESULT_NODE_COUNT, nodeCount); + + String message = (desc != null && !desc.isBlank()) + ? "Automation updated successfully. Description: \"" + desc.trim() + "\". " + nodeCount + " steps." + : "Automation updated successfully. " + nodeCount + " steps."; + summary.put(RESULT_MESSAGE, message); + } catch (Exception e) { + classLogger.warn("Could not build summary from generated doc", e); + summary.put(RESULT_MESSAGE, "Automation updated successfully."); + } + + return summary; + } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + return meta; + } + + @Override + public String getReactorDescription() { + return "Modifies an automation using AI generation without opening any UI. " + + "Chains BuildAutomation (edit mode) and SaveAutomation silently. " + + "Use EditAutomation instead when the user needs to review the change before saving."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) { + return "The project ID of the automation to edit."; + } else if (EDIT_DESCRIPTION_KEY.equals(key)) { + return "Plain-language description of the change to make. " + + "Example: 'Change the SQL filter to pull records from the last 7 days'. " + + "Call GetAutomationStructure first to understand the current automation structure."; + } + return super.getDescriptionForKey(key); + } } diff --git a/src/prerna/reactor/automation/README.md b/src/prerna/reactor/automation/README.md deleted file mode 100644 index 42b15e3420f..00000000000 --- a/src/prerna/reactor/automation/README.md +++ /dev/null @@ -1,71 +0,0 @@ -# Automation Engine - -Executes sequential node pipelines against SEMOSS engines. Users build a pipeline in the form editor (FE), save it as `automation.json`, then trigger it manually. Each run is tracked in the DB and the FE polls for progress. - -## How it works - -``` -FE calls runPixelAsync("TriggerAutomation(...)") - → Monolith spawns a virtual thread, returns jobId immediately - → FE polls GetActiveAutomationRun every 500ms (up to 10×) to get runId - -TriggerAutomationReactor (virtual thread, synchronous) - → reads automation.json (nodes in order) - → claims single-run slot (AUTOMATION_ACTIVE_RUN) → runId visible to FE - → inserts AUTOMATION_RUNS + AUTOMATION_NODE_OUTPUTS rows - → calls AutomationRunEngine.run() synchronously - → returns completed run result to jobId slot when done - -AutomationRunEngine (same virtual thread) - → iterates nodes in saved order - → dispatches each node to its IAutomationNodeExecutor - → writes node output + status to AUTOMATION_NODE_OUTPUTS after each node - → FE polls GetAutomationRun every 3s while runId is known -``` - -## Reactors - -| Reactor | Pixel | What it does | -| --- | --- | --- | -| `TriggerAutomationReactor` | `TriggerAutomation(project=["id"])` | Starts a run synchronously; returns completed run result | -| `GetActiveAutomationRunReactor` | `GetActiveAutomationRun(project=["id"])` | Returns `{RUN_ID, PROJECT_ID}` from active-run lock table; empty map when idle | -| `GetAutomationReactor` | `GetAutomation(project=["id"])` | Returns saved pipeline definition (automation.json) | -| `GetAutomationConfigReactor` | `GetAutomationConfig(project=["id"])` | Returns env var/secret config; masks sensitive values | -| `GetAutomationRunReactor` | `GetAutomationRun(project=["id"], runId=["id"])` | Returns live run state for FE polling | -| `ListAutomationRunsReactor` | `ListAutomationRuns(project=["id"])` | Returns run history | -| `CancelAutomationRunReactor` | `CancelAutomationRun(project=["id"], runId=["id"])` | Cancels an in-progress run | -| `SaveAutomationReactor` | `SaveAutomation(project=["id"], config=["{}"])` | Persists pipeline definition | -| `SaveAutomationConfigReactor` | `SaveAutomationConfig(project=["id"], config=["[]"])` | Persists env var config | -| `RunAutomationNodeReactor` | `RunAutomationNode(project=["id"], nodeId=["id"])` | Single-node test run; result not persisted | - -## Node types - -Each node type has a corresponding `IAutomationNodeExecutor` in `nodes/`: - -| Type | Executor | What it does | -| --- | --- | --- | -| `trigger` | (no executor) | Seed node; provides `triggered_at`, `date`, `run_id` scope vars | -| `database-engine` | `DatabaseEngineNodeExecutor` | Runs SQL via `SqlQuery` pixel | -| `model-engine` | `ModelEngineNodeExecutor` | LLM ask or embeddings via `IModelEngine` | -| `vector-engine` | `VectorEngineNodeExecutor` | Search, add, delete, list via `IVectorDatabaseEngine` | -| `storage-engine` | `StorageEngineNodeExecutor` | File operations via `IStorageEngine` | -| `function-engine` | `FunctionEngineNodeExecutor` | Function invocation via `IFunctionEngine` | -| `app` | `AppEngineNodeExecutor` | Arbitrary pixel, optionally scoped to a project | -| `wait` | `WaitNodeExecutor` | Sleep N seconds; cancel-aware | - -## DB tables - -| Table | Purpose | -| --- | --- | -| `AUTOMATION_ACTIVE_RUN` | PK on `PROJECT_ID` — enforces one concurrent run per project | -| `AUTOMATION_RUNS` | One row per run: status, timing, node counts | -| `AUTOMATION_NODE_OUTPUTS` | One row per node per run: status, output, preview, duration | - -## Key utilities - -| Class | Purpose | -| --- | --- | -| `AutomationExecutionUtils` | Shared statics: GSON, scope building, variable resolution, output transforms, preview generation | -| `AutomationDatabaseUtility` | All DB reads/writes for runs and node outputs | -| `PixelExecutionUtils` | Timeout-enforced pixel execution with ThreadStore propagation | -| `AutomationConstants` | String constants for all keys, statuses, and file names | diff --git a/src/prerna/reactor/automation/RunAutomationNodeReactor.java b/src/prerna/reactor/automation/RunAutomationNodeReactor.java index 8f8efffcb03..53dea62e563 100644 --- a/src/prerna/reactor/automation/RunAutomationNodeReactor.java +++ b/src/prerna/reactor/automation/RunAutomationNodeReactor.java @@ -27,6 +27,8 @@ *******************************************************************************/ package prerna.reactor.automation; +import prerna.reactor.automation.utils.AutomationExecutionUtils; + import java.util.HashMap; import java.util.List; import java.util.Map; @@ -39,7 +41,6 @@ import prerna.reactor.AbstractReactor; import prerna.reactor.agent.mcp.MCPUtility; import prerna.reactor.automation.nodes.AutomationNodeContext; -import prerna.reactor.automation.nodes.AutomationNodeExecutors; import prerna.reactor.automation.nodes.IAutomationNodeExecutor; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; @@ -56,7 +57,7 @@ public class RunAutomationNodeReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(RunAutomationNodeReactor.class); - // Not standardized in ReactorKeysEnum — matches the local-key convention used by + // Not standardized in ReactorKeysEnum - matches the local-key convention used by // prerna.reactor.agent (e.g. GetAgentRunReactor.RUN_ID_KEY). private static final String NODE_ID_KEY = "nodeId"; private static final String RUN_ID_KEY = "runId"; @@ -101,7 +102,7 @@ public NounMetadata execute() { if (AutomationConstants.NODE_TRIGGER.equals(type)) { rawOutput = scope.get(AutomationConstants.SCOPE_TRIGGERED_AT); } else { - IAutomationNodeExecutor executor = AutomationNodeExecutors.EXECUTORS.get(type); + IAutomationNodeExecutor executor = IAutomationNodeExecutor.EXECUTORS.get(type); if (executor == null) { throw new IllegalArgumentException("Unsupported node type: " + type); } @@ -153,7 +154,7 @@ private Map buildScope(String projectId, String contextRunId) { Map scope = AutomationExecutionUtils.buildInitialScope(null, this.insight.getUser()); if (contextRunId != null && !contextRunId.isEmpty()) { - // Scope the context run to this project — otherwise a user could pull node outputs + // Scope the context run to this project - otherwise a user could pull node outputs // (potentially containing other apps' secrets/data) from a run belonging to a // project they don't have access to by passing an arbitrary runId. Map contextRunDetail = AutomationDatabaseUtility.getRunDetail(contextRunId); @@ -178,14 +179,14 @@ private Map buildScope(String projectId, String contextRunId) { @Override public String getReactorDescription() { - return "Executes a single automation node in isolation for testing — result is not persisted."; + return "Executes a single automation node in isolation for testing - result is not persisted."; } @Override public Map getMcpToolMetadata() { Map meta = new HashMap<>(); // Node executors can perform real side effects (DB writes, storage uploads/deletes, - // arbitrary pixel execution) — requires explicit human confirmation. + // arbitrary pixel execution) - requires explicit human confirmation. meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); return meta; } diff --git a/src/prerna/reactor/automation/SaveAutomationConfigReactor.java b/src/prerna/reactor/automation/SaveAutomationConfigReactor.java index 21515dff289..7ba70edcc43 100644 --- a/src/prerna/reactor/automation/SaveAutomationConfigReactor.java +++ b/src/prerna/reactor/automation/SaveAutomationConfigReactor.java @@ -27,11 +27,14 @@ *******************************************************************************/ package prerna.reactor.automation; +import prerna.reactor.automation.utils.AutomationExecutionUtils; + import java.io.File; import java.io.IOException; import java.util.Base64; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -42,9 +45,8 @@ import com.google.gson.reflect.TypeToken; import prerna.auth.utils.SecurityProjectUtils; - - import prerna.reactor.AbstractReactor; +import prerna.util.Utility; import prerna.reactor.agent.mcp.MCPUtility; import prerna.sablecc2.om.PixelDataType; import prerna.sablecc2.om.PixelOperationType; @@ -61,6 +63,7 @@ public class SaveAutomationConfigReactor extends AbstractReactor { public SaveAutomationConfigReactor() { this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), ReactorKeysEnum.CONFIG.getKey() }; + this.keyRequired = new int[] { 1, 0 }; } @Override @@ -79,14 +82,29 @@ public NounMetadata execute() { } String config; + if (configEncoded == null || configEncoded.isEmpty()) { + config = AutomationConstants.EMPTY_JSON_ARRAY; + } else { + try { + config = new String(Base64.getDecoder().decode(configEncoded.trim()), StandardCharsets.UTF_8); + } catch (Exception e) { + // configEncoded was not Base64 - treat as raw JSON and validate it parses below + config = configEncoded; + } + } + // Validate JSON is parseable before writing anything try { - config = new String(Base64.getDecoder().decode(configEncoded != null ? configEncoded : AutomationConstants.EMPTY_JSON_ARRAY), StandardCharsets.UTF_8); + AutomationExecutionUtils.GSON.fromJson(config, LIST_OF_MAP_TYPE); } catch (Exception e) { - config = configEncoded != null ? configEncoded : AutomationConstants.EMPTY_JSON_ARRAY; + throw new IllegalArgumentException("config must be valid JSON or Base64-encoded JSON"); } String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); - File configFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_CONFIG_FILE_NAME); + File configFile = Paths.get(portalsFolder, AutomationConstants.AUTOMATION_CONFIG_FILE_NAME).toFile(); + String normalizedConfigPath = Utility.normalizePath(configFile.getAbsolutePath()); + if (!normalizedConfigPath.startsWith(portalsFolder)) { + throw new IllegalArgumentException("Invalid file path"); + } // GetAutomationConfig masks sensitive values as SENSITIVE_MASK. // Restore the real values from disk so a config round-trip cannot silently destroy them. diff --git a/src/prerna/reactor/automation/SaveAutomationReactor.java b/src/prerna/reactor/automation/SaveAutomationReactor.java index c9260c6c2e0..329f095a576 100644 --- a/src/prerna/reactor/automation/SaveAutomationReactor.java +++ b/src/prerna/reactor/automation/SaveAutomationReactor.java @@ -32,6 +32,7 @@ import java.util.Base64; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -88,7 +89,11 @@ public NounMetadata execute() { IProject project = Utility.getProject(projectId); String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); - File automationFile = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); + File automationFile = Paths.get(portalsFolder, AutomationConstants.AUTOMATION_FILE_NAME).toFile(); + String normalizedPath = Utility.normalizePath(automationFile.getAbsolutePath()); + if (!normalizedPath.startsWith(portalsFolder)) { + throw new IllegalArgumentException("Invalid file path"); + } try { automationFile.getParentFile().mkdirs(); diff --git a/src/prerna/reactor/automation/TriggerAutomationReactor.java b/src/prerna/reactor/automation/TriggerAutomationReactor.java index f898fe49fb2..741d7df0791 100644 --- a/src/prerna/reactor/automation/TriggerAutomationReactor.java +++ b/src/prerna/reactor/automation/TriggerAutomationReactor.java @@ -27,6 +27,8 @@ *******************************************************************************/ package prerna.reactor.automation; +import prerna.reactor.automation.utils.AutomationExecutionUtils; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -121,26 +123,11 @@ public NounMetadata execute() { // Build final result in the same shape as GetAutomationRunReactor Map runDetail = AutomationDatabaseUtility.getRunDetail(runId); List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(runId); - List> nodeResults = new ArrayList<>(); + List> nodeResults = AutomationDatabaseUtility.buildNodeResults(nodeOutputs); int completedCount = 0; - if (nodeOutputs != null) { - for (Map output : nodeOutputs) { - Map nodeResult = new HashMap<>(); - nodeResult.put(AutomationConstants.NODE_ID, output.get(AutomationConstants.NODE_ID)); - nodeResult.put(AutomationConstants.NODE_LABEL, output.get(AutomationConstants.NODE_LABEL)); - nodeResult.put(AutomationConstants.STATUS, output.get(AutomationConstants.STATUS)); - nodeResult.put(AutomationConstants.DURATION_MS, output.get(AutomationConstants.DURATION_MS)); - String outputForDisplay = (String) output.get(AutomationConstants.OUTPUT_VALUE); - if (outputForDisplay == null || outputForDisplay.isBlank()) { - outputForDisplay = (String) output.get(AutomationConstants.OUTPUT_PREVIEW); - } - nodeResult.put(AutomationConstants.OUTPUT_PREVIEW, outputForDisplay); - nodeResult.put(AutomationConstants.OUTPUT_VALUE, output.get(AutomationConstants.OUTPUT_VALUE)); - nodeResult.put(AutomationConstants.ERROR_MESSAGE, output.get(AutomationConstants.ERROR_MESSAGE)); - nodeResults.add(nodeResult); - if (AutomationConstants.NODE_STATUS_SUCCESS.equals(output.get(AutomationConstants.STATUS))) { - completedCount++; - } + for (Map nodeResult : nodeResults) { + if (AutomationConstants.NODE_STATUS_SUCCESS.equals(nodeResult.get(AutomationConstants.STATUS))) { + completedCount++; } } // Trigger nodes succeed immediately in the engine but never write a SUCCESS diff --git a/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java index 514a5aa89be..9e0bbbbba8c 100644 --- a/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java @@ -36,8 +36,8 @@ import prerna.om.ThreadStore; import prerna.project.api.IProject; import prerna.reactor.automation.AutomationConstants; -import prerna.reactor.automation.AutomationExecutionUtils; -import prerna.reactor.automation.PixelExecutionUtils; +import prerna.reactor.automation.utils.AutomationExecutionUtils; +import prerna.reactor.automation.utils.PixelExecutionUtils; import prerna.util.Utility; /** @@ -77,6 +77,7 @@ public Object execute(AutomationNodeContext ctx) throws Exception { classLogger.debug("App-engine node \"{}\" executing pixel in appId={}", nodeLabel, resolvedAppId != null ? resolvedAppId : "caller context"); + int timeoutMs = AutomationExecutionUtils.getNodeTimeout(ctx.node()); if (resolvedAppId != null && !resolvedAppId.isBlank()) { resolvedAppId = SecurityProjectUtils.testUserProjectIdForAlias(ctx.insight().getUser(), resolvedAppId); if (!SecurityProjectUtils.userCanEditProject(ctx.insight().getUser(), resolvedAppId)) { @@ -91,13 +92,13 @@ public Object execute(AutomationNodeContext ctx) throws Exception { ThreadStore.setContextProjectIdOverride(resolvedAppId); ThreadStore.setContextProjectNameOverride(project.getProjectName()); try { - return PixelExecutionUtils.runAndCollect(ctx.insight(), resolvedPixel); + return PixelExecutionUtils.runAndCollect(ctx.insight(), resolvedPixel, timeoutMs); } finally { ThreadStore.clearContextProjectOverride(); } } - return PixelExecutionUtils.runAndCollect(ctx.insight(), resolvedPixel); + return PixelExecutionUtils.runAndCollect(ctx.insight(), resolvedPixel, timeoutMs); } } diff --git a/src/prerna/reactor/automation/nodes/AutomationNodeContext.java b/src/prerna/reactor/automation/nodes/AutomationNodeContext.java index ceca4f5b620..09bc55bb08a 100644 --- a/src/prerna/reactor/automation/nodes/AutomationNodeContext.java +++ b/src/prerna/reactor/automation/nodes/AutomationNodeContext.java @@ -27,6 +27,8 @@ *******************************************************************************/ package prerna.reactor.automation.nodes; +import prerna.reactor.automation.utils.AutomationExecutionUtils; + import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @@ -34,15 +36,20 @@ import prerna.reactor.automation.AutomationConstants; /** - * Immutable param bundle passed to every {@link IAutomationNodeExecutor}. + * Param bundle passed to every {@link IAutomationNodeExecutor}. * *

Consolidates all per-node execution inputs so executor implementations have a single, * typed object to work with rather than long parameter lists. * + *

Note: {@code scope} is intentionally mutable and shared across all node executions within + * one run. Each node may add output variables to {@code scope} after it completes successfully, + * making those values available to subsequent nodes via {@code ${varName}} substitution. + * * @param runId the automation run ID (use {@code "test"} for single-node test runs) * @param projectId the owning project UUID * @param node the raw node definition map from {@code automation.json} - * @param scope mutable variable scope — string key→value pairs resolved by {@code ${varName}} + * @param scope mutable variable scope — string key→value pairs resolved by {@code ${varName}}; + * shared across all nodes in the run and updated after each node completes * @param configMap project automation config key→value pairs resolved by {@code ${config.KEY}} * @param insight caller's {@link Insight} context (carries user, session, pixel engine) * @param cancelFlag shared flag the executor checks to honour mid-node cancellation requests @@ -69,6 +76,13 @@ public String nodeType() { return (String) node.get(AutomationConstants.NODE_FIELD_TYPE); } + /** + * Returns the node's config map, or an empty map if absent. + * + *

The cast to {@code Map} is safe: Gson always deserializes JSON objects + * as {@code Map}, and the automation document is loaded exclusively via Gson + * in {@link prerna.reactor.automation.AutomationExecutionUtils#loadAutomationDoc}. + */ @SuppressWarnings("unchecked") public Map config() { Object config = node.get(AutomationConstants.NODE_FIELD_CONFIG); diff --git a/src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java b/src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java deleted file mode 100644 index 52160781277..00000000000 --- a/src/prerna/reactor/automation/nodes/AutomationNodeExecutors.java +++ /dev/null @@ -1,51 +0,0 @@ -/******************************************************************************* - * Copyright 2015 Defense Health Agency (DHA) - * - * If your use of this software does not include any GPLv2 components: - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * ---------------------------------------------------------------------------- - * If your use of this software includes any GPLv2 components: - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 2 - * of the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - *******************************************************************************/ -package prerna.reactor.automation.nodes; - -import java.util.Map; - -import prerna.reactor.automation.AutomationConstants; - -/** - * Shared registry of stateless node executor instances. - * Used by both TriggerAutomationReactor and RunAutomationNodeReactor. - */ -public final class AutomationNodeExecutors { - - private AutomationNodeExecutors() {} - - public static final Map EXECUTORS = Map.of( - AutomationConstants.NODE_WAIT, new WaitNodeExecutor(), - AutomationConstants.NODE_DATABASE_ENGINE, new DatabaseEngineNodeExecutor(), - AutomationConstants.NODE_MODEL_ENGINE, new ModelEngineNodeExecutor(), - AutomationConstants.NODE_VECTOR_ENGINE, new VectorEngineNodeExecutor(), - AutomationConstants.NODE_STORAGE_ENGINE, new StorageEngineNodeExecutor(), - AutomationConstants.NODE_FUNCTION_ENGINE, new FunctionEngineNodeExecutor(), - AutomationConstants.NODE_APP, new AppEngineNodeExecutor() - ); -} diff --git a/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java index 6dfb6e3e937..85b85821f82 100644 --- a/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java @@ -33,8 +33,8 @@ import org.apache.logging.log4j.Logger; import prerna.reactor.automation.AutomationConstants; -import prerna.reactor.automation.AutomationExecutionUtils; -import prerna.reactor.automation.PixelExecutionUtils; +import prerna.reactor.automation.utils.AutomationExecutionUtils; +import prerna.reactor.automation.utils.PixelExecutionUtils; /** * Executor for {@code database-engine} nodes. Runs a SQL query against a configured database engine diff --git a/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java index 2d7b09f41ae..a93641f5aa5 100644 --- a/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java @@ -32,15 +32,27 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.google.gson.reflect.TypeToken; - import prerna.auth.utils.SecurityEngineUtils; import prerna.auth.utils.SecurityQueryUtils; import prerna.engine.api.IFunctionEngine; import prerna.reactor.automation.AutomationConstants; -import prerna.reactor.automation.AutomationExecutionUtils; +import prerna.reactor.automation.utils.AutomationExecutionUtils; import prerna.util.Utility; +/** + * Executes a function-engine automation node. Delegates to the platform + * {@link prerna.engine.api.IFunctionEngine} API to run arbitrary server-side functions. + * + *

Config fields (from {@code node.config}): + *

    + *
  • {@code engineId} (required) — UUID or alias of the target function engine
    • + *
  • {@code params} (optional) — JSON object of parameters to pass to the function; supports + * {@code ${var}} substitution; defaults to {@code {}}
    • + *
+ * + *

Requires edit access to {@code engineId} because function execution runs arbitrary + * server-side code, which is a mutating/write operation. + */ public final class FunctionEngineNodeExecutor implements IAutomationNodeExecutor { private static final Logger classLogger = LogManager.getLogger(FunctionEngineNodeExecutor.class); @@ -59,9 +71,9 @@ public Object execute(AutomationNodeContext ctx) throws Exception { String resolvedParams = AutomationExecutionUtils.resolve(params, scope, configMap); resolvedEngineId = SecurityQueryUtils.testUserEngineIdForAlias(ctx.insight().getUser(), resolvedEngineId); - if (!SecurityEngineUtils.userCanViewEngine(ctx.insight().getUser(), resolvedEngineId)) { + if (!SecurityEngineUtils.userCanEditEngine(ctx.insight().getUser(), resolvedEngineId)) { throw new IllegalArgumentException( - "Function-engine node \"" + nodeLabel + "\": engine does not exist or user does not have access: " + resolvedEngineId); + "Function-engine node \"" + nodeLabel + "\": engine does not exist or user does not have edit access: " + resolvedEngineId); } IFunctionEngine engine = Utility.getFunctionEngine(resolvedEngineId); @@ -79,7 +91,7 @@ private static Map parseParams(String json, String nodeLabel) { if (json == null || json.isBlank()) return Map.of(); try { Map parsed = AutomationExecutionUtils.GSON.fromJson(json, - new TypeToken>() {}.getType()); + AutomationExecutionUtils.MAP_TYPE); return parsed != null ? parsed : Map.of(); } catch (Exception e) { throw new IllegalArgumentException("Function-engine node \"" + nodeLabel + "\": params is not valid JSON: " + e.getMessage(), e); diff --git a/src/prerna/reactor/automation/nodes/IAutomationNodeExecutor.java b/src/prerna/reactor/automation/nodes/IAutomationNodeExecutor.java index 77ae0a1b50e..1f9aacd37a9 100644 --- a/src/prerna/reactor/automation/nodes/IAutomationNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/IAutomationNodeExecutor.java @@ -27,13 +27,15 @@ *******************************************************************************/ package prerna.reactor.automation.nodes; +import java.util.Map; + +import prerna.reactor.automation.AutomationConstants; + /** * Executes one automation node's operation, given all the context that node type needs. * *

One concrete implementation per {@code node.type} value (e.g. {@code WaitNodeExecutor}, - * {@code DatabaseEngineNodeExecutor}), resolved via a - * {@code Map} registry in - * {@link prerna.reactor.automation.TriggerAutomationReactor#executeSingleNode} instead of the + * {@code DatabaseEngineNodeExecutor}), resolved via {@link #EXECUTORS} instead of the * previous {@code if/else} chain keyed on {@code type}. * *

Mirrors the shape already established in this codebase for "one conceptual operation, many @@ -47,13 +49,32 @@ */ public interface IAutomationNodeExecutor { + /** + * Shared registry of stateless node executor instances, keyed by {@code node.type} value. + * Used by both {@link prerna.reactor.automation.AutomationRunEngine} and + * {@link prerna.reactor.automation.RunAutomationNodeReactor}. + * + *

Static interface fields are implicitly {@code public static final} in Java. + * + * + */ + Map EXECUTORS = Map.of( + AutomationConstants.NODE_WAIT, new WaitNodeExecutor(), + AutomationConstants.NODE_DATABASE_ENGINE, new DatabaseEngineNodeExecutor(), + AutomationConstants.NODE_MODEL_ENGINE, new ModelEngineNodeExecutor(), + AutomationConstants.NODE_VECTOR_ENGINE, new VectorEngineNodeExecutor(), + AutomationConstants.NODE_STORAGE_ENGINE, new StorageEngineNodeExecutor(), + AutomationConstants.NODE_FUNCTION_ENGINE, new FunctionEngineNodeExecutor(), + AutomationConstants.NODE_APP, new AppEngineNodeExecutor() + ); + /** * Executes this node's operation and returns its raw output - the same shape previously * returned by each {@code executeXNode} method (a String, a JSON string, or a * {@code Map} that the caller will serialize). The caller - * ({@code executeSingleNode}) is responsible for applying the node's output transform, - * generating the preview, and checkpointing success/failure to the database - executors - * should not do any of that themselves. + * ({@link prerna.reactor.automation.AutomationRunEngine#executeSingleNode}) is responsible for + * applying the node's output transform, generating the preview, and checkpointing + * success/failure to the database - executors should not do any of that themselves. * * @param ctx the node's execution context - node definition, scope, config, and callbacks * for recursing into sibling/child nodes where the node type requires it diff --git a/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java index 9b890de637c..6fbfcf49daa 100644 --- a/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java @@ -30,6 +30,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -40,7 +41,7 @@ import prerna.engine.impl.model.responses.AskModelEngineResponse; import prerna.engine.impl.model.responses.EmbeddingsModelEngineResponse; import prerna.reactor.automation.AutomationConstants; -import prerna.reactor.automation.AutomationExecutionUtils; +import prerna.reactor.automation.utils.AutomationExecutionUtils; import prerna.util.Utility; /** @@ -78,7 +79,10 @@ public Object execute(AutomationNodeContext ctx) throws Exception { case AutomationConstants.OP_EMBEDDINGS: { String values = NodeConfigHelper.required(config, AutomationConstants.CONFIG_VALUES, nodeLabel); String resolvedValues = AutomationExecutionUtils.resolve(values, scope, configMap); - List valueList = Arrays.asList(resolvedValues.split(",")); + List valueList = Arrays.stream(resolvedValues.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); EmbeddingsModelEngineResponse response = engine.embeddings(valueList, ctx.insight(), null); return response.getResponse(); } diff --git a/src/prerna/reactor/automation/nodes/NodeConfigHelper.java b/src/prerna/reactor/automation/nodes/NodeConfigHelper.java index ea7930dfe37..265ac432235 100644 --- a/src/prerna/reactor/automation/nodes/NodeConfigHelper.java +++ b/src/prerna/reactor/automation/nodes/NodeConfigHelper.java @@ -29,72 +29,43 @@ import java.util.Map; +import prerna.reactor.automation.utils.AutomationExecutionUtils; + /** - * Shared config-extraction helpers used by all automation node executors. + * Delegates to {@link AutomationExecutionUtils} for config-extraction helpers. + * + *

The canonical implementations now live in {@code AutomationExecutionUtils}. This class + * is kept as a thin delegation shim so existing callers in the {@code nodes} sub-package + * continue to compile without modification. It will be removed in a later cleanup pass once + * those callers are updated to reference {@code AutomationExecutionUtils} directly. * - *

Each executor previously contained private copies of {@code required()}, - * {@code optional()}, and {@code optionalInt()} with identical logic. This class - * centralises them so changes (e.g. to error message format) propagate everywhere. + * @deprecated Use {@link AutomationExecutionUtils#required}, {@link AutomationExecutionUtils#optional}, + * and {@link AutomationExecutionUtils#optionalInt} directly. */ +@Deprecated final class NodeConfigHelper { private NodeConfigHelper() { // utility class } - /** - * Returns the string value of {@code key} from {@code config}, throwing if absent or blank. - * - * @param config node config map - * @param key config key to look up - * @param nodeLabel display label of the containing node (used in the error message) - */ + /** @see AutomationExecutionUtils#required(Map, String, String) */ static String required(Map config, String key, String nodeLabel) { - Object v = config.get(key); - if (v == null || v.toString().isBlank()) { - throw new IllegalArgumentException( - "Node \"" + nodeLabel + "\": '" + key + "' is required"); - } - return v.toString(); + return AutomationExecutionUtils.required(config, key, nodeLabel); } - /** - * Returns the string value of {@code key} from {@code config}, or {@code def} if absent or blank. - * - * @param config node config map - * @param key config key to look up - * @param def value to return when key is absent or blank - */ + /** @see AutomationExecutionUtils#optional(Map, String, String) */ static String optional(Map config, String key, String def) { - Object v = config.get(key); - return (v == null || v.toString().isBlank()) ? def : v.toString(); + return AutomationExecutionUtils.optional(config, key, def); } - /** - * Returns the string value of {@code key} from {@code config}, or {@code null} if absent or blank. - * - * @param config node config map - * @param key config key to look up - */ + /** @see AutomationExecutionUtils#optional(Map, String) */ static String optional(Map config, String key) { - return optional(config, key, null); + return AutomationExecutionUtils.optional(config, key); } - /** - * Returns the integer value of {@code key} from {@code config}, or {@code def} if absent, blank, - * or not parseable as an integer. - * - * @param config node config map - * @param key config key to look up - * @param def default value when key is absent, blank, or unparseable - */ + /** @see AutomationExecutionUtils#optionalInt(Map, String, int) */ static int optionalInt(Map config, String key, int def) { - Object v = config.get(key); - if (v == null) return def; - try { - return Integer.parseInt(v.toString().trim()); - } catch (NumberFormatException e) { - return def; - } + return AutomationExecutionUtils.optionalInt(config, key, def); } } diff --git a/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java index 2734e062eac..2ee0d18b268 100644 --- a/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java @@ -38,9 +38,23 @@ import prerna.auth.utils.SecurityQueryUtils; import prerna.engine.api.IStorageEngine; import prerna.reactor.automation.AutomationConstants; -import prerna.reactor.automation.AutomationExecutionUtils; +import prerna.reactor.automation.utils.AutomationExecutionUtils; import prerna.util.Utility; +/** + * Executes a storage-engine automation node. Supports file transfer ({@code download}, + * {@code upload}), deletion ({@code delete}), in-memory base64 reads ({@code read-base64}), + * and directory listing ({@code list}) via the platform {@link prerna.engine.api.IStorageEngine} API. + * + *

Config fields (from {@code node.config}): + *

    + *
  • {@code engineId} (required) — UUID or alias of the target storage engine
    • + *
  • {@code operation} (optional) — one of {@code list} (default), {@code download}, + * {@code upload}, {@code delete}, {@code read-base64}
    • + *
  • {@code storagePath} (required for most operations) — path within the storage engine
    • + *
  • {@code filePath} (required for download/upload) — local file system path
    • + *
+ */ public final class StorageEngineNodeExecutor implements IAutomationNodeExecutor { private static final Logger classLogger = LogManager.getLogger(StorageEngineNodeExecutor.class); diff --git a/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java index 73bc450edce..92ac2d21dfe 100644 --- a/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java @@ -30,6 +30,7 @@ import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -38,9 +39,25 @@ import prerna.auth.utils.SecurityQueryUtils; import prerna.engine.api.IVectorDatabaseEngine; import prerna.reactor.automation.AutomationConstants; -import prerna.reactor.automation.AutomationExecutionUtils; +import prerna.reactor.automation.utils.AutomationExecutionUtils; import prerna.util.Utility; +/** + * Executes a vector-engine automation node. Supports document ingestion ({@code add-file}, + * {@code add-csv}), deletion ({@code delete}), listing ({@code list}), and semantic search + * ({@code search}) via the platform {@link prerna.engine.api.IVectorDatabaseEngine} API. + * + *

Config fields (from {@code node.config}): + *

    + *
  • {@code engineId} (required) — UUID or alias of the target vector engine
    • + *
  • {@code operation} (optional) — one of {@code search} (default), {@code add-file}, + * {@code add-csv}, {@code delete}, {@code list}
    • + *
  • {@code filePath} (required for add-file/add-csv) — comma-separated file paths
    • + *
  • {@code fileNames} (required for delete) — comma-separated document names to remove
    • + *
  • {@code command} (required for search) — the query string
    • + *
  • {@code limit} (optional for search) — max results; default {@value AutomationConstants#DEFAULT_VECTOR_SEARCH_LIMIT}
    • + *
+ */ public final class VectorEngineNodeExecutor implements IAutomationNodeExecutor { private static final Logger classLogger = LogManager.getLogger(VectorEngineNodeExecutor.class); @@ -79,7 +96,10 @@ public Object execute(AutomationNodeContext ctx) throws Exception { case AutomationConstants.OP_ADD_CSV: { String filePaths = NodeConfigHelper.required(config, AutomationConstants.CONFIG_FILE_PATH, nodeLabel); String resolvedPaths = AutomationExecutionUtils.resolve(filePaths, scope, configMap); - List paths = Arrays.asList(resolvedPaths.split(",")); + List paths = Arrays.stream(resolvedPaths.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); engine.addDocument(paths, null); return "Added " + paths.size() + " file(s)"; } @@ -90,7 +110,10 @@ public Object execute(AutomationNodeContext ctx) throws Exception { case AutomationConstants.OP_DELETE: { String fileNames = NodeConfigHelper.required(config, AutomationConstants.CONFIG_FILE_NAMES, nodeLabel); String resolvedNames = AutomationExecutionUtils.resolve(fileNames, scope, configMap); - List names = Arrays.asList(resolvedNames.split(",")); + List names = Arrays.stream(resolvedNames.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); engine.removeDocument(names, null); return "Deleted " + names.size() + " file(s)"; } diff --git a/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java index 6b609e3b749..320937e4305 100644 --- a/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java @@ -36,7 +36,7 @@ import prerna.reactor.automation.AutomationCancelledException; import prerna.reactor.automation.AutomationConstants; import prerna.reactor.automation.AutomationDatabaseUtility; -import prerna.reactor.automation.AutomationExecutionUtils; +import prerna.reactor.automation.utils.AutomationExecutionUtils; /** * Executes a "wait" node: sleeps for the configured number of seconds. @@ -58,9 +58,8 @@ public Object execute(AutomationNodeContext ctx) throws Exception { Map config = ctx.config(); String nodeLabel = ctx.nodeLabel(); - String secondsTemplate = config.get(AutomationConstants.CONFIG_SECONDS) != null - ? config.get(AutomationConstants.CONFIG_SECONDS).toString() - : String.valueOf(AutomationConstants.WAIT_DEFAULT_SECONDS); + String secondsTemplate = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_SECONDS, + String.valueOf(AutomationConstants.WAIT_DEFAULT_SECONDS)); String resolved = AutomationExecutionUtils.resolve(secondsTemplate, ctx.scope(), ctx.configMap()); int seconds; diff --git a/src/prerna/reactor/automation/AutomationExecutionUtils.java b/src/prerna/reactor/automation/utils/AutomationExecutionUtils.java similarity index 66% rename from src/prerna/reactor/automation/AutomationExecutionUtils.java rename to src/prerna/reactor/automation/utils/AutomationExecutionUtils.java index f7272cee1ae..a970cd67586 100644 --- a/src/prerna/reactor/automation/AutomationExecutionUtils.java +++ b/src/prerna/reactor/automation/utils/AutomationExecutionUtils.java @@ -25,7 +25,7 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. *******************************************************************************/ -package prerna.reactor.automation; +package prerna.reactor.automation.utils; import java.io.File; import java.io.IOException; @@ -41,8 +41,6 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -50,8 +48,6 @@ import org.apache.commons.text.StringSubstitutor; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.json.JSONArray; -import org.json.JSONObject; import com.google.gson.Gson; import com.google.gson.GsonBuilder; @@ -60,21 +56,14 @@ import com.google.gson.reflect.TypeToken; import prerna.auth.User; -import prerna.auth.utils.SecurityEngineUtils; -import prerna.engine.api.IDatabaseEngine; -import prerna.engine.api.IEngine; -import prerna.reactor.agent.mcp.MCPUtility; -import prerna.engine.api.IRDBMSEngine; -import prerna.project.api.IProject; import prerna.util.AssetUtility; -import prerna.util.Constants; -import prerna.util.Utility; +import prerna.reactor.automation.AutomationConstants; /** * Shared static utilities for the automation execution engine. * - *

Centralizes logic shared across {@link TriggerAutomationReactor} and - * {@link RunAutomationNodeReactor}. + *

Centralizes logic shared across {@link prerna.reactor.automation.TriggerAutomationReactor} and + * {@link prerna.reactor.automation.RunAutomationNodeReactor}. */ public final class AutomationExecutionUtils { @@ -90,7 +79,7 @@ public final class AutomationExecutionUtils { private static final Type LIST_OBJ_MAP_TYPE = new TypeToken>>() {}.getType(); /** - * Shared Gson instance for the whole automation engine — public so the + * Shared Gson instance for the whole automation engine - public so the * {@code nodes} sub-package has one shared instance to reuse instead of each * file declaring its own. */ @@ -274,9 +263,9 @@ private static String transformJsonPath(String rawStr, String path) { /** * Normalises any of the three dataset formats into a canonical {@code {headers, values}} map: - * 1. List<Map> (rows-as-objects) — produced by {@link DatabaseEngineNodeExecutor} - * 2. {@code {data: {headers, values}}} — SEMOSS wrapped envelope - * 3. {@code {headers, values}} — SEMOSS direct + * 1. List<Map> (rows-as-objects) - produced by {@link DatabaseEngineNodeExecutor} + * 2. {@code {data: {headers, values}}} - SEMOSS wrapped envelope + * 3. {@code {headers, values}} - SEMOSS direct */ @SuppressWarnings("unchecked") private static Map extractDataset(Object parsed) { @@ -316,7 +305,7 @@ private static Map extractDataset(Object parsed) { return null; } - /** Parses JSON to Object — returns List for arrays, Map for objects (handles all executor output shapes). */ + /** Parses JSON to Object - returns List for arrays, Map for objects (handles all executor output shapes). */ private static Object parseJsonAny(String json) { if (json == null || json.isBlank()) return null; try { @@ -342,7 +331,7 @@ private static Map parseJson(String json) { * {@code triggered_at}, and {@code run_id} (when non-blank). * * @param runId the run ID to seed into scope, or {@code null} for test runs - * @param user the triggering user — used to localise {@code date} and {@code triggered_at} + * @param user the triggering user - used to localise {@code date} and {@code triggered_at} * to the user's configured timezone; falls back to UTC when {@code null} or * when no zone has been set on the user */ @@ -489,7 +478,7 @@ public static Map applyPlaygroundInputs(List /** * Builds a stable MCP parameter name for a playground-fillable node field. - * Slugifies the node label (lowercase, non-alphanumeric chars → underscore, collapsed) + * Slugifies the node label (lowercase, non-alphanumeric chars -> underscore, collapsed) * and appends the field name. Falls back to {@code "input_" + fieldName} if label is blank. * * @param nodeLabel the node's display label (may be null or empty) @@ -536,273 +525,89 @@ public static Map coerceToMap(Object raw) { return new HashMap<>(); } - // -- LLM helpers (shared by GenerateAutomationReactor and ExplainAutomationReactor) --- + // -- Node config access helpers ----------------------------------------------- + // These mirror the methods in NodeConfigHelper (nodes sub-package), which now + // delegates to these. New callers should use AutomationExecutionUtils directly. /** - * Returns the ID of the first MODEL-type engine the user has access to, or {@code null} if none. - */ - public static String findFirstModelEngine(User user) { - try { - List> engines = SecurityEngineUtils.getUserEngineList( - user, List.of("MODEL"), null, false, null, null, null, "1", "0", null); - if (engines != null && !engines.isEmpty()) { - Object id = engines.get(0).get("database_id"); - return id != null ? String.valueOf(id) : null; - } - } catch (Exception e) { - classLogger.warn("Failed to auto-discover model engine", e); - } - return null; - } - - /** - * Extracts the text content from a model engine response map. - * Checks {@code response}, {@code output}, and {@code content} keys in order. - */ - public static String extractResponseText(Map response) { - if (response == null) return null; - Object resp = response.get("response"); - if (resp instanceof String s && !s.isBlank()) return s; - Object output = response.get("output"); - if (output instanceof String s && !s.isBlank()) return s; - Object content = response.get("content"); - if (content instanceof String s && !s.isBlank()) return s; - return null; - } - - // -- Generation helpers (shared by GenerateAutomationReactor and BuildAutomationReactor) --- - - /** - * Builds the Room options map for RunAgent — registers all user-accessible engines - * as MCP tools and includes the room's built-in tools. + * Returns the string value of {@code key} from {@code config}, throwing if absent or blank. + * + * @param config node config map + * @param key config key to look up + * @param nodeLabel display label of the containing node (used in the error message) */ - public static Map buildEngineMcpOptions(User user, String systemPrompt) { - List> mcpList = new ArrayList<>(); - try { - List> engines = SecurityEngineUtils.getUserEngineList( - user, GENERATION_ENGINE_TYPES, null, false, null, null, null, "50", "0", null); - if (engines != null) { - for (Map engine : engines) { - String id = engine.getOrDefault("database_id", "").toString(); - String name = engine.getOrDefault("database_name", "").toString(); - if (id.isBlank()) continue; - try { - IEngine engineObj = Utility.getEngine(id); - if (engineObj == null || !engineObj.isMCPEnabled()) continue; - } catch (Exception e) { - classLogger.debug("Skipping engine {} from MCP list — could not load: {}", id, e.getMessage()); - continue; - } - Map entry = new HashMap<>(); - entry.put("id", id); - entry.put("name", name); - entry.put("type", "ENGINE"); - mcpList.add(entry); - } - } - } catch (Exception e) { - classLogger.warn("Failed to build MCP engine list for automation room", e); - } - Map dbMakerEntry = new HashMap<>(); - dbMakerEntry.put("id", Constants.MCP_DATABASE_MAKER); - dbMakerEntry.put("name", "Database Tools"); - dbMakerEntry.put("type", "PROJECT"); - mcpList.add(dbMakerEntry); - Map roomEntry = new HashMap<>(); - roomEntry.put("id", MCPUtility.ROOM_MCP_ID); - roomEntry.put("name", MCPUtility.ROOM_MCP_NAME); - roomEntry.put("type", MCPUtility.ROOM_MCP_TYPE); - roomEntry.put("fromRoom", true); - mcpList.add(roomEntry); - Map options = new HashMap<>(); - if (systemPrompt != null && !systemPrompt.isBlank()) { - options.put("instructions", systemPrompt); + public static String required(Map config, String key, String nodeLabel) { + Object v = config.get(key); + if (v == null || v.toString().isBlank()) { + throw new IllegalArgumentException( + "Node \"" + nodeLabel + "\": '" + key + "' is required"); } - options.put("mcp", mcpList); - return options; + return v.toString(); } - /** Engine types listed in the generation prompt so the LLM can populate engineId fields. */ - static final List GENERATION_ENGINE_TYPES = Arrays.asList("DATABASE", "MODEL", "VECTOR", "STORAGE", "FUNCTION"); - /** - * Builds the "## Available engines" prompt section for generation — lists each engine the - * current user can access so the LLM can assign engineId fields correctly. + * Returns the string value of {@code key} from {@code config}, or {@code def} if absent or blank. + * + * @param config node config map + * @param key config key to look up + * @param def value to return when key is absent or blank */ - static String buildAvailableEnginesSection(User user) { - StringBuilder sb = new StringBuilder("## Available engines\n"); - try { - List> engines = SecurityEngineUtils.getUserEngineList( - user, GENERATION_ENGINE_TYPES, null, false, null, null, null, "50", "0", null); - if (engines == null || engines.isEmpty()) { - sb.append("None available — leave engineId as empty string.\n"); - return sb.toString(); - } - for (Map engine : engines) { - String id = engine.getOrDefault("database_id", "").toString(); - String name = engine.getOrDefault("database_name", "").toString(); - String type = engine.getOrDefault("engine_type", "").toString().toUpperCase(); - sb.append("- type=").append(type) - .append(" id=\"").append(id) - .append("\" name=\"").append(name).append("\""); - // For relational databases, include the table list so the model can match - // the user's intent to the right DB without needing to call get_db_schema first. - if ("DATABASE".equals(type) && !id.isBlank()) { - try { - IDatabaseEngine dbEngine = Utility.getDatabase(id); - if (dbEngine instanceof IRDBMSEngine rdbms) { - List tables = rdbms.getPixelConcepts(); - if (tables != null && !tables.isEmpty()) { - sb.append(" tables=["); - for (int i = 0; i < tables.size(); i++) { - String table = tables.get(i); - // Strip the "ConceptName__" prefix that pixel concepts sometimes include - int sep = table.lastIndexOf("__"); - sb.append(sep >= 0 ? table.substring(sep + 2) : table); - if (i < tables.size() - 1) sb.append(", "); - } - sb.append("]"); - } - } - } catch (Exception e) { - // Table list is best-effort; the model can still call get_db_schema if needed - } - } - sb.append("\n"); - } - } catch (Exception e) { - classLogger.warn("Failed to build engine list for generation prompt", e); - sb.append("(engine list unavailable — leave engineId as empty string)\n"); - } - return sb.toString(); + public static String optional(Map config, String key, String def) { + Object v = config.get(key); + return (v == null || v.toString().isBlank()) ? def : v.toString(); } /** - * Fetches the table/column schema (with data types) for the given list of database engine IDs. - * Formatted as a prompt section for LLM use. + * Returns the string value of {@code key} from {@code config}, or {@code null} if absent or blank. + * + * @param config node config map + * @param key config key to look up */ - static String buildSchemaForEngineIds(List engineIds) { - StringBuilder sb = new StringBuilder("## Database schema\n"); - for (String engineId : engineIds) { - try { - IDatabaseEngine dbEngine = Utility.getDatabase(engineId); - if (!(dbEngine instanceof IRDBMSEngine rdbms)) { - sb.append("Database id=\"").append(engineId).append("\": (non-relational — no table schema)\n"); - continue; - } - sb.append("Database id=\"").append(engineId).append("\":\n"); - List tables = rdbms.getPixelConcepts(); - if (tables == null || tables.isEmpty()) { - sb.append(" (no tables found)\n"); - continue; - } - for (String table : tables) { - List columns = rdbms.getPixelSelectors(table); - sb.append(" - ").append(table).append(" ("); - if (columns != null && !columns.isEmpty()) { - StringBuilder cols = new StringBuilder(); - for (String col : columns) { - int sep = col.lastIndexOf("__"); - String colName = sep >= 0 ? col.substring(sep + 2) : col; - String dataType = null; - try { - String physicalUri = rdbms.getPhysicalUriFromPixelSelector(col); - if (physicalUri != null) { - dataType = rdbms.getDataTypes(physicalUri); - } - } catch (Exception ignored) { - // Data type fetch is best-effort; column name alone is still useful - } - if (cols.length() > 0) cols.append(", "); - cols.append(colName); - if (dataType != null && !dataType.isBlank()) { - cols.append(": ").append(dataType.trim()); - } - } - sb.append(cols); - } - sb.append(")\n"); - } - } catch (Exception e) { - classLogger.warn("Failed to fetch schema for engine {}", engineId, e); - sb.append("Database id=\"").append(engineId).append("\": (schema unavailable)\n"); - } - } - return sb.toString(); + public static String optional(Map config, String key) { + return optional(config, key, null); } /** - * Returns the list of custom reactors available in the given project, formatted as a prompt - * section for LLM use. Used to fill in app-node pixel expressions. + * Returns the integer value of {@code key} from {@code config}, or {@code def} if absent, blank, + * or not parseable as an integer. + * + * @param config node config map + * @param key config key to look up + * @param def default value when key is absent, blank, or unparseable */ - static String buildReactorListSection(String projectId) { - StringBuilder sb = new StringBuilder("## Available custom reactors in this project\n"); + public static int optionalInt(Map config, String key, int def) { + Object v = config.get(key); + if (v == null) return def; try { - IProject project = Utility.getProject(projectId); - if (project == null) { - sb.append("(project not found — leave app node pixel as-is)\n"); - return sb.toString(); - } - java.util.TreeSet reactors = project.getAvailableReactors(); - if (reactors == null || reactors.isEmpty()) { - sb.append("(none — leave app node pixel as a comment placeholder)\n"); - return sb.toString(); - } - for (String name : reactors) { - sb.append("- ").append(name).append("\n"); - } - } catch (Exception e) { - classLogger.warn("Failed to build reactor list for project {}", projectId, e); - sb.append("(reactor list unavailable — leave app node pixel as a comment placeholder)\n"); + return Integer.parseInt(v.toString().trim()); + } catch (NumberFormatException e) { + return def; } - return sb.toString(); } - /** - * Strips leading/trailing markdown code fences ({@code ```json ... ```} or {@code ``` ... ```}) - * that models sometimes add despite being instructed not to. - */ - static String stripCodeFences(String raw) { - String trimmed = raw.strip(); - if (trimmed.startsWith("```")) { - int firstNewline = trimmed.indexOf('\n'); - if (firstNewline != -1) { - trimmed = trimmed.substring(firstNewline + 1); - } - if (trimmed.endsWith("```")) { - trimmed = trimmed.substring(0, trimmed.lastIndexOf("```")).strip(); - } - } - return trimmed; - } + // -- Automation document loading ----------------------------------------------- /** - * Validates that a generated document string is parseable JSON and contains the minimum - * required structure ({@code graph.nodes} must be non-empty). Throws on failure. + * Reads a project's {@code automation.json} as a raw JSON string. + * Returns a minimal blank document string when the file does not exist or cannot be read, + * rather than throwing. Suitable for callers (e.g. LLM prompts) that need a fallback value. + * + * @param projectId the project to load the automation doc for + * @return the file contents, or {@code {"version":1,"graph":{"nodes":[],"edges":[]}}} when absent */ - static void validateGeneratedDoc(String raw) { + public static String loadAutomationDocOrEmpty(String projectId) { try { - JSONObject doc = new JSONObject(raw); - if (!doc.has("graph")) { - throw new IllegalStateException("Generated document is missing the 'graph' field."); - } - JSONObject graph = doc.getJSONObject("graph"); - if (!graph.has("nodes")) { - throw new IllegalStateException("Generated graph is missing the 'nodes' array."); - } - JSONArray nodes = graph.getJSONArray("nodes"); - if (nodes.length() == 0) { - throw new IllegalStateException("Generated graph has no nodes."); + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File f = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); + if (f.exists() && f.isFile()) { + return Files.readString(f.toPath(), StandardCharsets.UTF_8); } - } catch (org.json.JSONException e) { - classLogger.warn("Generated automation doc is not valid JSON (truncated): {}", raw.length() > 500 ? raw.substring(0, 500) + "..." : raw, e); - throw new IllegalStateException( - "The AI model returned an invalid response. Please try again with a different description.", e); + } catch (IOException e) { + classLogger.warn("Could not read automation.json for project {} - returning empty doc", projectId, e); } + return "{\"version\":1,\"graph\":{\"nodes\":[],\"edges\":[]}}"; } - // -- Automation document loading ----------------------------------------------- - /** * Loads and parses a project's {@code automation.json} (graph + trigger config). * Throws if the file is missing or unreadable. @@ -823,4 +628,3 @@ public static Map loadAutomationDoc(String projectId) { } } - diff --git a/src/prerna/reactor/automation/utils/AutomationGenerationUtils.java b/src/prerna/reactor/automation/utils/AutomationGenerationUtils.java new file mode 100644 index 00000000000..f24c0c5dc62 --- /dev/null +++ b/src/prerna/reactor/automation/utils/AutomationGenerationUtils.java @@ -0,0 +1,334 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.utils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.engine.api.IDatabaseEngine; +import prerna.engine.api.IEngine; +import prerna.engine.api.IRDBMSEngine; +import prerna.project.api.IProject; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.util.Constants; +import prerna.util.Utility; +import prerna.reactor.automation.AutomationConstants; + +/** + * LLM/generation helpers used exclusively by the AI reactor group: + * {@link prerna.reactor.automation.BuildAutomationReactor}, + * {@link prerna.reactor.automation.ExplainAutomationReactor}, {@link prerna.reactor.automation.GenerateNodeLabelReactor}, + * {@link prerna.reactor.automation.GenerateRunSummaryReactor}, {@link prerna.reactor.automation.QuickEditAutomationReactor}, + * {@link prerna.reactor.automation.AutomationAskRoomReactor}, and {@link prerna.reactor.automation.GetReactorSignatureReactor}. + * + *

Execution-time helpers (resolve, scope building, output transforms, etc.) + * live in {@link AutomationExecutionUtils}. + */ +public final class AutomationGenerationUtils { + + private static final Logger classLogger = LogManager.getLogger(AutomationGenerationUtils.class); + + /** Engine types listed in the generation prompt so the LLM can populate engineId fields. */ + static final List GENERATION_ENGINE_TYPES = Arrays.asList("DATABASE", "MODEL", "VECTOR", "STORAGE", "FUNCTION"); + + private AutomationGenerationUtils() {} + + /** + * Returns the ID of the first MODEL-type engine the user has access to, or {@code null} if none. + */ + public static String findFirstModelEngine(User user) { + try { + List> engines = SecurityEngineUtils.getUserEngineList( + user, List.of("MODEL"), null, false, null, null, null, "1", "0", null); + if (engines != null && !engines.isEmpty()) { + Object id = engines.get(0).get("database_id"); + return id != null ? String.valueOf(id) : null; + } + } catch (Exception e) { + classLogger.warn("Failed to auto-discover model engine", e); + } + return null; + } + + /** + * Extracts the text content from a model engine response map. + * Checks {@code response}, {@code output}, and {@code content} keys in order. + */ + public static String extractResponseText(Map response) { + if (response == null) return null; + Object resp = response.get("response"); + if (resp instanceof String s && !s.isBlank()) return s; + Object output = response.get("output"); + if (output instanceof String s && !s.isBlank()) return s; + Object content = response.get("content"); + if (content instanceof String s && !s.isBlank()) return s; + return null; + } + + /** + * Builds the Room options map for RunAgent - registers all user-accessible engines + * as MCP tools and includes the room's built-in tools. + */ + public static Map buildEngineMcpOptions(User user, String systemPrompt) { + List> mcpList = new ArrayList<>(); + try { + List> engines = SecurityEngineUtils.getUserEngineList( + user, GENERATION_ENGINE_TYPES, null, false, null, null, null, "50", "0", null); + if (engines != null) { + for (Map engine : engines) { + String id = engine.getOrDefault("database_id", "").toString(); + String name = engine.getOrDefault("database_name", "").toString(); + if (id.isBlank()) continue; + try { + IEngine engineObj = Utility.getEngine(id); + if (engineObj == null || !engineObj.isMCPEnabled()) continue; + } catch (Exception e) { + classLogger.debug("Skipping engine {} from MCP list - could not load: {}", id, e.getMessage()); + continue; + } + Map entry = new HashMap<>(); + entry.put("id", id); + entry.put("name", name); + entry.put("type", "ENGINE"); + mcpList.add(entry); + } + } + } catch (Exception e) { + classLogger.warn("Failed to build MCP engine list for automation room", e); + } + Map dbMakerEntry = new HashMap<>(); + dbMakerEntry.put("id", Constants.MCP_DATABASE_MAKER); + dbMakerEntry.put("name", "Database Tools"); + dbMakerEntry.put("type", "PROJECT"); + mcpList.add(dbMakerEntry); + Map roomEntry = new HashMap<>(); + roomEntry.put("id", MCPUtility.ROOM_MCP_ID); + roomEntry.put("name", MCPUtility.ROOM_MCP_NAME); + roomEntry.put("type", MCPUtility.ROOM_MCP_TYPE); + roomEntry.put("fromRoom", true); + mcpList.add(roomEntry); + Map options = new HashMap<>(); + if (systemPrompt != null && !systemPrompt.isBlank()) { + options.put("instructions", systemPrompt); + } + options.put("mcp", mcpList); + return options; + } + + /** + * Builds the "## Available engines" prompt section for generation - lists each engine the + * current user can access so the LLM can assign engineId fields correctly. + */ + public static String buildAvailableEnginesSection(User user) { + StringBuilder sb = new StringBuilder("## Available engines\n"); + try { + List> engines = SecurityEngineUtils.getUserEngineList( + user, GENERATION_ENGINE_TYPES, null, false, null, null, null, "50", "0", null); + if (engines == null || engines.isEmpty()) { + sb.append("None available - leave engineId as empty string.\n"); + return sb.toString(); + } + for (Map engine : engines) { + String id = engine.getOrDefault("database_id", "").toString(); + String name = engine.getOrDefault("database_name", "").toString(); + String type = engine.getOrDefault("engine_type", "").toString().toUpperCase(); + sb.append("- type=").append(type) + .append(" id=\"").append(id) + .append("\" name=\"").append(name).append("\""); + // For relational databases, include the table list so the model can match + // the user's intent to the right DB without needing to call get_db_schema first. + if ("DATABASE".equals(type) && !id.isBlank()) { + try { + IDatabaseEngine dbEngine = Utility.getDatabase(id); + if (dbEngine instanceof IRDBMSEngine rdbms) { + List tables = rdbms.getPixelConcepts(); + if (tables != null && !tables.isEmpty()) { + sb.append(" tables=["); + for (int i = 0; i < tables.size(); i++) { + String table = tables.get(i); + // Strip the "ConceptName__" prefix that pixel concepts sometimes include + int sep = table.lastIndexOf("__"); + sb.append(sep >= 0 ? table.substring(sep + 2) : table); + if (i < tables.size() - 1) sb.append(", "); + } + sb.append("]"); + } + } + } catch (Exception e) { + // Table list is best-effort; the model can still call get_db_schema if needed + } + } + sb.append("\n"); + } + } catch (Exception e) { + classLogger.warn("Failed to build engine list for generation prompt", e); + sb.append("(engine list unavailable - leave engineId as empty string)\n"); + } + return sb.toString(); + } + + /** + * Fetches the table/column schema (with data types) for the given list of database engine IDs. + * Formatted as a prompt section for LLM use. + */ + public static String buildSchemaForEngineIds(List engineIds) { + StringBuilder sb = new StringBuilder("## Database schema\n"); + for (String engineId : engineIds) { + try { + IDatabaseEngine dbEngine = Utility.getDatabase(engineId); + if (!(dbEngine instanceof IRDBMSEngine rdbms)) { + sb.append("Database id=\"").append(engineId).append("\": (non-relational - no table schema)\n"); + continue; + } + sb.append("Database id=\"").append(engineId).append("\":\n"); + List tables = rdbms.getPixelConcepts(); + if (tables == null || tables.isEmpty()) { + sb.append(" (no tables found)\n"); + continue; + } + for (String table : tables) { + List columns = rdbms.getPixelSelectors(table); + sb.append(" - ").append(table).append(" ("); + if (columns != null && !columns.isEmpty()) { + StringBuilder cols = new StringBuilder(); + for (String col : columns) { + int sep = col.lastIndexOf("__"); + String colName = sep >= 0 ? col.substring(sep + 2) : col; + String dataType = null; + try { + String physicalUri = rdbms.getPhysicalUriFromPixelSelector(col); + if (physicalUri != null) { + dataType = rdbms.getDataTypes(physicalUri); + } + } catch (Exception ignored) { + // Data type fetch is best-effort; column name alone is still useful + } + if (cols.length() > 0) cols.append(", "); + cols.append(colName); + if (dataType != null && !dataType.isBlank()) { + cols.append(": ").append(dataType.trim()); + } + } + sb.append(cols); + } + sb.append(")\n"); + } + } catch (Exception e) { + classLogger.warn("Failed to fetch schema for engine {}", engineId, e); + sb.append("Database id=\"").append(engineId).append("\": (schema unavailable)\n"); + } + } + return sb.toString(); + } + + /** + * Returns the list of custom reactors available in the given project, formatted as a prompt + * section for LLM use. Used to fill in app-node pixel expressions. + */ + public static String buildReactorListSection(String projectId) { + StringBuilder sb = new StringBuilder("## Available custom reactors in this project\n"); + try { + IProject project = Utility.getProject(projectId); + if (project == null) { + sb.append("(project not found - leave app node pixel as-is)\n"); + return sb.toString(); + } + java.util.TreeSet reactors = project.getAvailableReactors(); + if (reactors == null || reactors.isEmpty()) { + sb.append("(none - leave app node pixel as a comment placeholder)\n"); + return sb.toString(); + } + for (String name : reactors) { + sb.append("- ").append(name).append("\n"); + } + } catch (Exception e) { + classLogger.warn("Failed to build reactor list for project {}", projectId, e); + sb.append("(reactor list unavailable - leave app node pixel as a comment placeholder)\n"); + } + return sb.toString(); + } + + /** + * Strips leading/trailing markdown code fences ({@code ```json ... ```} or {@code ``` ... ```}) + * that models sometimes add despite being instructed not to. + */ + public static String stripCodeFences(String raw) { + String trimmed = raw.strip(); + if (trimmed.startsWith("```")) { + int firstNewline = trimmed.indexOf('\n'); + if (firstNewline != -1) { + trimmed = trimmed.substring(firstNewline + 1); + } + if (trimmed.endsWith("```")) { + trimmed = trimmed.substring(0, trimmed.lastIndexOf("```")).strip(); + } + } + return trimmed; + } + + /** + * Validates that a generated document string is parseable JSON and contains the minimum + * required structure ({@code graph.nodes} must be non-empty). Throws on failure. + * + *

Uses Gson for parsing (consistent with the rest of the automation package) rather than + * {@code org.json}. Gson deserializes JSON objects as {@code Map} and arrays + * as {@code List}, so the casts below are safe for well-formed JSON. + */ + public static void validateGeneratedDoc(String raw) { + Map doc; + try { + doc = AutomationExecutionUtils.GSON.fromJson(raw, AutomationExecutionUtils.MAP_TYPE); + } catch (Exception e) { + classLogger.warn("Generated automation doc is not valid JSON (truncated): {}", + raw.length() > 500 ? raw.substring(0, 500) + "..." : raw, e); + throw new IllegalStateException( + "The AI model returned an invalid response. Please try again with a different description.", e); + } + if (doc == null || !(doc.get("graph") instanceof Map)) { + throw new IllegalStateException("Generated document is missing the 'graph' field."); + } + @SuppressWarnings("unchecked") + Map graph = (Map) doc.get("graph"); + if (!(graph.get("nodes") instanceof List)) { + throw new IllegalStateException("Generated graph is missing the 'nodes' array."); + } + @SuppressWarnings("unchecked") + List nodes = (List) graph.get("nodes"); + if (nodes.isEmpty()) { + throw new IllegalStateException("Generated graph has no nodes."); + } + } +} diff --git a/src/prerna/reactor/automation/PixelExecutionUtils.java b/src/prerna/reactor/automation/utils/PixelExecutionUtils.java similarity index 85% rename from src/prerna/reactor/automation/PixelExecutionUtils.java rename to src/prerna/reactor/automation/utils/PixelExecutionUtils.java index f8869b828e7..b4a192a5721 100644 --- a/src/prerna/reactor/automation/PixelExecutionUtils.java +++ b/src/prerna/reactor/automation/utils/PixelExecutionUtils.java @@ -25,7 +25,7 @@ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. *******************************************************************************/ -package prerna.reactor.automation; +package prerna.reactor.automation.utils; import java.util.HashMap; import java.util.List; @@ -46,6 +46,7 @@ import prerna.sablecc2.om.PixelOperationType; import prerna.sablecc2.om.nounmeta.NounMetadata; import prerna.sablecc2.om.task.ITask; +import prerna.reactor.automation.AutomationConstants; /** * Shared utility for executing Pixel expressions and returning clean, serializable results. @@ -121,7 +122,11 @@ private static NounMetadata executeWithTimeout(Insight insight, String pixel, in try { Callable task = () -> { if (contextSnapshot != null && !contextSnapshot.isEmpty()) { - ThreadStore.getInsightId(); + // ThreadStore.setThreadMapObject calls CURRENT.get() directly, which returns + // null on a fresh worker thread and would NPE. Call setInsightId("") first + // to force initialization of the ThreadLocal map; setThreadMapObject then + // overwrites all values including insightId with the captured context. + ThreadStore.setInsightId(""); ThreadStore.setThreadMapObject(contextSnapshot); } try { @@ -209,4 +214,23 @@ public String getPixel() { return pixel; } } + + /** + * Thrown mid-node when a cancellation request is detected during a blocking operation. + * + *

Using a distinct unchecked exception type (rather than a flag return value or a checked + * exception) lets nodes that loop internally - such as {@code WaitNodeExecutor} sleeping in + * chunks - abort cleanly without threading a cancellation result through every call frame. The + * caller ({@code AutomationRunEngine.executeSingleNode}) catches this type specifically and + * records the run as {@link AutomationConstants#STATUS_CANCELLED} instead of + * {@link AutomationConstants#STATUS_FAILED}, so the end-user sees the correct terminal state. + */ + public static class AutomationCancelledException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public AutomationCancelledException(String message) { + super(message); + } + } } diff --git a/src/prerna/reactor/scheduler/SchedulerConstants.java b/src/prerna/reactor/scheduler/SchedulerConstants.java index 2c82484ca2c..f79530be26f 100644 --- a/src/prerna/reactor/scheduler/SchedulerConstants.java +++ b/src/prerna/reactor/scheduler/SchedulerConstants.java @@ -128,7 +128,6 @@ private SchedulerConstants() { public static final String VARCHAR_250 = "VARCHAR (250)"; public static final String VARCHAR_255 = "VARCHAR (255)"; public static final String VARCHAR_512 = "VARCHAR (512)"; - public static final String VARCHAR_1000 = "VARCHAR (1000)"; public static final String VARCHAR_2000 = "VARCHAR (2000)"; public static final String INTEGER = "INTEGER"; diff --git a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java index 0d53c406fab..bfbcf0a13db 100644 --- a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java +++ b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java @@ -124,8 +124,6 @@ import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_8; import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_80; import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_95; -import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_2000; - import java.util.ArrayList; import java.util.Arrays; @@ -133,7 +131,6 @@ import prerna.engine.impl.owl.AbstractOwlCreator; import prerna.engine.impl.owl.WriteOWLEngine; -import prerna.reactor.automation.AutomationConstants; public class SchedulerOwlCreator extends AbstractOwlCreator { @@ -277,42 +274,6 @@ public void createColumnsAndTypes() { Pair.with(EXEC_ID, VARCHAR_200), Pair.with(JOB_ID, VARCHAR_200), Pair.with(JOB_GROUP, VARCHAR_200))); - - addTable(AutomationConstants.TABLE_AUTOMATION_RUNS, Arrays.asList( - Pair.with(AutomationConstants.RUN_ID, VARCHAR_255), - Pair.with(AutomationConstants.PROJECT_ID, VARCHAR_255), - Pair.with(AutomationConstants.AUTOMATION_ID, VARCHAR_255), - Pair.with(AutomationConstants.STATUS, VARCHAR_200), - Pair.with(AutomationConstants.TRIGGER_TYPE, VARCHAR_200), - Pair.with(AutomationConstants.STARTED_AT, TIMESTAMP), - Pair.with(AutomationConstants.COMPLETED_AT, TIMESTAMP), - Pair.with(AutomationConstants.FAILED_NODE_ID, VARCHAR_255), - Pair.with(AutomationConstants.ERROR_MESSAGE, CLOB), - Pair.with(AutomationConstants.LAST_HEARTBEAT, TIMESTAMP), - Pair.with(AutomationConstants.TOTAL_NODES, INTEGER), - Pair.with(AutomationConstants.COMPLETED_NODES, INTEGER), - Pair.with(AutomationConstants.CREATED_BY, VARCHAR_255), - Pair.with(AutomationConstants.CANCEL_REQUESTED, BOOLEAN), - Pair.with(AutomationConstants.RESULT_SUMMARY_COL, VARCHAR_2000))); - - addTable(AutomationConstants.TABLE_AUTOMATION_NODE_OUTPUTS, Arrays.asList( - Pair.with(AutomationConstants.RUN_ID, VARCHAR_255), - Pair.with(AutomationConstants.NODE_ID, VARCHAR_255), - Pair.with(AutomationConstants.NODE_LABEL, VARCHAR_512), - Pair.with(AutomationConstants.EXECUTION_ORDER, INTEGER), - Pair.with(AutomationConstants.STATUS, VARCHAR_200), - Pair.with(AutomationConstants.STARTED_AT, TIMESTAMP), - Pair.with(AutomationConstants.COMPLETED_AT, TIMESTAMP), - Pair.with(AutomationConstants.DURATION_MS, BIGINT), - Pair.with(AutomationConstants.OUTPUT_VAR, VARCHAR_255), - Pair.with(AutomationConstants.OUTPUT_VALUE, CLOB), - Pair.with(AutomationConstants.OUTPUT_PREVIEW, VARCHAR_2000), - Pair.with(AutomationConstants.ERROR_MESSAGE, CLOB))); - - addTable(AutomationConstants.TABLE_AUTOMATION_ACTIVE_RUN, Arrays.asList( - Pair.with(AutomationConstants.PROJECT_ID, VARCHAR_255), - Pair.with(AutomationConstants.RUN_ID, VARCHAR_255), - Pair.with(AutomationConstants.CLAIMED_AT, TIMESTAMP))); // @formatter:on } diff --git a/src/prerna/util/SMSSWebWatcher.java b/src/prerna/util/SMSSWebWatcher.java index e9ae6df68b6..5defd38c90c 100644 --- a/src/prerna/util/SMSSWebWatcher.java +++ b/src/prerna/util/SMSSWebWatcher.java @@ -213,15 +213,12 @@ public void init() { try { SystemEngineRegistry.loadSystemEngine(folderToWatch + "/" + fileNames[schedulerDbNameIndex]); SchedulerDatabaseUtility.startServer(); - } catch (Exception e) { - classLogger.error("Failed to load and start the scheduler database", e); - } - - try { + // Automation tables live in the scheduler DB, so only initialize them + // after the scheduler DB has started successfully. AutomationDatabaseUtility.initialize(); AutomationDatabaseUtility.markStaleRunsInterrupted(); } catch (Exception e) { - classLogger.error("Failed to initialize automation engine tables", e); + classLogger.error("Failed to load and start the scheduler database", e); } } } From 2d7137fa25d8ead4fe9296a3406644e2c3bd7bd6 Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Thu, 13 Aug 2026 15:34:19 -0400 Subject: [PATCH 22/25] feat: fix small code issues and add create automation --- .../automation/CreateAutomationReactor.java | 77 ++++++++++++------- .../automation/GenerateRunSummaryReactor.java | 4 + .../GetAutomationSchemaReactor.java | 14 +++- .../nodes/StorageEngineNodeExecutor.java | 71 ++++++++++++++++- .../automation/utils/PixelExecutionUtils.java | 53 +++++++++++-- .../reactor/project/CreateProjectReactor.java | 5 ++ 6 files changed, 184 insertions(+), 40 deletions(-) diff --git a/src/prerna/reactor/automation/CreateAutomationReactor.java b/src/prerna/reactor/automation/CreateAutomationReactor.java index 154007c5f86..43e70997f71 100644 --- a/src/prerna/reactor/automation/CreateAutomationReactor.java +++ b/src/prerna/reactor/automation/CreateAutomationReactor.java @@ -29,6 +29,8 @@ import prerna.reactor.automation.utils.PixelExecutionUtils; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; @@ -36,6 +38,9 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import prerna.auth.User; +import prerna.project.api.IProject; +import prerna.project.impl.ProjectHelper; import prerna.reactor.AbstractReactor; import prerna.reactor.agent.mcp.MCPUtility; import prerna.sablecc2.om.PixelDataType; @@ -44,7 +49,7 @@ /** * Creates a new automation project and returns its ID so the LLM can immediately chain to - * {@code EditAutomation} to interactively build it. + * {@code QuickEditAutomation} to interactively build it. * *

The project name must start with a letter and contain only letters, numbers, and spaces * (enforced by {@code CreateProject}). @@ -52,7 +57,7 @@ *

Typical LLM flow: *

    *
  1. LLM calls {@code CreateAutomation(projectName=["My Automation"])} — auto, no UI
    2. - *
  2. LLM immediately chains {@code EditAutomation(project=[""], instruction=["..."])}
    3. + *
  3. LLM immediately chains {@code QuickEditAutomation(project=[""], editDescription=["..."])}
    4. *
  4. User sees the editor open with the AI-generated draft
    5. *
* @@ -93,33 +98,28 @@ public NounMetadata execute() { "Project name must start with a letter and contain only letters, numbers, and spaces. Got: " + projectName); } - // Delegate to CreateProject — it handles project scaffolding and security. - // CODE project type matches all existing automation projects. - String createPixel = String.format( - "CreateProject(project=[\"%s\"], projectType=[\"CODE\"], global=[false]);", - projectName); - - Object raw; - try { - raw = PixelExecutionUtils.runAndCollect(this.insight, createPixel); - } catch (PixelExecutionUtils.AutomationPixelException e) { - classLogger.error("CreateProject pixel error for '{}'", projectName, e); - throw new IllegalArgumentException("Failed to create project '" + projectName + "': " + e.getMessage()); - } - - if (!(raw instanceof Map)) { - classLogger.error("CreateProject returned unexpected result type for '{}': {}", - projectName, raw == null ? "null" : raw.getClass().getName()); - throw new IllegalArgumentException("Unexpected response from CreateProject for: " + projectName); + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You must be signed in to create an automation."); } - @SuppressWarnings("unchecked") - Map projectData = (Map) raw; - String projectId = (String) projectData.get("project_id"); + IProject project = ProjectHelper.generateNewProject(projectName, IProject.PROJECT_TYPE.AUTOMATION, + false, null, null, user, classLogger); + String projectId = project.getProjectId(); + String definition = buildStarterDefinition(); + String encodedDefinition = Base64.getEncoder().encodeToString(definition.getBytes(StandardCharsets.UTF_8)); + String encodedConfig = Base64.getEncoder().encodeToString( + AutomationConstants.EMPTY_JSON_ARRAY.getBytes(StandardCharsets.UTF_8)); - if (projectId == null || projectId.isBlank()) { - classLogger.error("CreateProject did not return a project_id for '{}'", projectName); - throw new IllegalArgumentException("Project was created but no project ID was returned for: " + projectName); + try { + PixelExecutionUtils.runAndCollect(this.insight, String.format( + "SaveAutomation(project=[\"%s\"], json=[\"%s\"]);", projectId, encodedDefinition)); + PixelExecutionUtils.runAndCollect(this.insight, String.format( + "SaveAutomationConfig(project=[\"%s\"], config=[\"%s\"]);", projectId, encodedConfig)); + } catch (PixelExecutionUtils.AutomationPixelException e) { + classLogger.error("Failed to scaffold automation project '{}'", projectName, e); + throw new IllegalStateException( + "Automation project was created but its starter assets could not be scaffolded: " + e.getMessage(), e); } classLogger.info("CreateAutomationReactor: created project '{}' with id {}", projectName, projectId); @@ -127,15 +127,34 @@ public NounMetadata execute() { Map result = new LinkedHashMap<>(); result.put(RESULT_SUCCESS, true); result.put(RESULT_PROJECT_ID, projectId); + result.put("project_id", projectId); result.put(RESULT_PROJECT_NAME, projectName); result.put(RESULT_MESSAGE, "Created automation project \"" + projectName + "\" (id: " + projectId + "). " - + "Call EditAutomation(project=[\"" + projectId + "\"], instruction=[\"\"]) " - + "to open the editor and build the automation."); + + "Call QuickEditAutomation(project=[\"" + projectId + + "\"], editDescription=[\"\"]) to build the automation."); return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); } + private static String buildStarterDefinition() { + Map trigger = new LinkedHashMap<>(); + trigger.put(AutomationConstants.NODE_FIELD_ID, "trigger"); + trigger.put(AutomationConstants.NODE_FIELD_TYPE, AutomationConstants.NODE_TRIGGER); + trigger.put(AutomationConstants.NODE_FIELD_LABEL, "Start"); + trigger.put(AutomationConstants.NODE_FIELD_CONFIG, Map.of()); + + Map graph = new LinkedHashMap<>(); + graph.put(AutomationConstants.DOC_NODES, java.util.List.of(trigger)); + graph.put(AutomationConstants.DOC_EDGES, java.util.List.of()); + + Map definition = new LinkedHashMap<>(); + definition.put(AutomationConstants.DOC_VERSION, AutomationConstants.DOC_CURRENT_VERSION); + definition.put(AutomationConstants.DOC_DESCRIPTION, ""); + definition.put(AutomationConstants.DOC_GRAPH, graph); + return prerna.reactor.automation.utils.AutomationExecutionUtils.GSON.toJson(definition); + } + @Override public Map getMcpToolMetadata() { Map meta = new HashMap<>(); @@ -146,7 +165,7 @@ public Map getMcpToolMetadata() { @Override public String getReactorDescription() { return "Creates a new blank automation project and returns its ID. " - + "Immediately chain EditAutomation with the returned project ID to interactively build the automation. " + + "Immediately chain QuickEditAutomation with the returned project ID to build the automation. " + "Project names must start with a letter and contain only letters, numbers, and spaces."; } diff --git a/src/prerna/reactor/automation/GenerateRunSummaryReactor.java b/src/prerna/reactor/automation/GenerateRunSummaryReactor.java index 8765dade622..00bd7f36412 100644 --- a/src/prerna/reactor/automation/GenerateRunSummaryReactor.java +++ b/src/prerna/reactor/automation/GenerateRunSummaryReactor.java @@ -96,6 +96,10 @@ public NounMetadata execute() { if (runDetail == null) { throw new IllegalArgumentException("Run not found: " + runId); } + Object runProjectId = runDetail.get(AutomationConstants.PROJECT_ID); + if (!projectId.equals(runProjectId)) { + throw new IllegalArgumentException("Run not found: " + runId); + } // Guard against summarizing an in-progress run - the result would be incomplete. Object runStatus = runDetail.get(AutomationConstants.STATUS); diff --git a/src/prerna/reactor/automation/GetAutomationSchemaReactor.java b/src/prerna/reactor/automation/GetAutomationSchemaReactor.java index 16561d0cc5f..f4351ef8e26 100644 --- a/src/prerna/reactor/automation/GetAutomationSchemaReactor.java +++ b/src/prerna/reactor/automation/GetAutomationSchemaReactor.java @@ -37,6 +37,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; import prerna.auth.utils.SecurityProjectUtils; import prerna.engine.api.IDatabaseEngine; import prerna.reactor.AbstractReactor; @@ -66,12 +68,16 @@ public GetAutomationSchemaReactor() { @Override public NounMetadata execute() { organizeKeys(); + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } String projectId = this.keyValue.get(this.keysToGet[0]); if (projectId == null || projectId.isBlank()) { throw new IllegalArgumentException("Must provide a project id"); } - projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); - if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { + projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { throw new IllegalArgumentException("Project does not exist or user does not have access"); } @@ -101,6 +107,10 @@ public NounMetadata execute() { if (engineIdObj == null || engineIdObj.toString().isBlank()) continue; String engineId = engineIdObj.toString(); + if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { + throw new IllegalArgumentException( + "Database engine configured for node '" + nodeLabel + "' is not accessible."); + } Map nodeSchema = buildNodeSchema(nodeLabel, engineId); if (nodeSchema != null) { result.add(nodeSchema); diff --git a/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java index 2ee0d18b268..ed655ff4290 100644 --- a/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java +++ b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java @@ -27,6 +27,10 @@ *******************************************************************************/ package prerna.reactor.automation.nodes; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Base64; import java.util.List; import java.util.Map; @@ -39,6 +43,7 @@ import prerna.engine.api.IStorageEngine; import prerna.reactor.automation.AutomationConstants; import prerna.reactor.automation.utils.AutomationExecutionUtils; +import prerna.util.AssetUtility; import prerna.util.Utility; /** @@ -52,12 +57,14 @@ *
  • {@code operation} (optional) — one of {@code list} (default), {@code download}, * {@code upload}, {@code delete}, {@code read-base64}
    • *
  • {@code storagePath} (required for most operations) — path within the storage engine
    • - *
  • {@code filePath} (required for download/upload) — local file system path
    • + *
  • {@code filePath} (required for download/upload) — relative path under the owning + * project's automation runtime directory
    • * */ public final class StorageEngineNodeExecutor implements IAutomationNodeExecutor { private static final Logger classLogger = LogManager.getLogger(StorageEngineNodeExecutor.class); + private static final String AUTOMATION_RUNTIME_FILES_FOLDER = "automation-files"; @Override public Object execute(AutomationNodeContext ctx) throws Exception { @@ -92,7 +99,8 @@ public Object execute(AutomationNodeContext ctx) throws Exception { String storagePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); String filePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_FILE_PATH, nodeLabel); String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); - String resolvedFile = AutomationExecutionUtils.resolve(filePath, scope, configMap); + String resolvedFile = resolveLocalPath(ctx.projectId(), + AutomationExecutionUtils.resolve(filePath, scope, configMap)); engine.copyToLocal(resolvedStorage, resolvedFile); return "Downloaded: " + resolvedStorage; } @@ -100,9 +108,10 @@ public Object execute(AutomationNodeContext ctx) throws Exception { String storagePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); String filePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_FILE_PATH, nodeLabel); String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); - String resolvedFile = AutomationExecutionUtils.resolve(filePath, scope, configMap); + String resolvedFile = resolveLocalPath(ctx.projectId(), + AutomationExecutionUtils.resolve(filePath, scope, configMap)); engine.copyToStorage(resolvedFile, resolvedStorage, null); - return "Uploaded: " + resolvedFile; + return "Uploaded: " + filePath; } case AutomationConstants.OP_DELETE: { String storagePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); @@ -126,4 +135,58 @@ public Object execute(AutomationNodeContext ctx) throws Exception { } } + /** + * Resolves a configured local path inside a project-owned runtime directory. + * + *

    Automation definitions must never grant arbitrary local server file-system access. Both + * upload and download operations are therefore confined to + * {@code /automation-files}. Existing symbolic links are rejected so a path + * within that directory cannot escape it. + * + * @param projectId owning automation project + * @param configuredPath relative user-configured path + * @return normalized absolute path inside the project runtime directory + * @throws IOException if the runtime directory cannot be created + */ + private static String resolveLocalPath(String projectId, String configuredPath) throws IOException { + if (configuredPath == null || configuredPath.isBlank()) { + throw new IllegalArgumentException("Storage-engine node local path must not be empty"); + } + if (configuredPath.matches("^[A-Za-z]:[\\\\/].*")) { + throw new IllegalArgumentException("Storage-engine node local path must be relative to the automation workspace"); + } + + Path configured = Paths.get(configuredPath); + if (configured.isAbsolute()) { + throw new IllegalArgumentException("Storage-engine node local path must be relative to the automation workspace"); + } + + Path workspace = Paths.get(AssetUtility.getProjectAppRootFolder(projectId), + AUTOMATION_RUNTIME_FILES_FOLDER).toAbsolutePath().normalize(); + Files.createDirectories(workspace); + + Path resolved = workspace.resolve(configured).normalize(); + if (!resolved.startsWith(workspace)) { + throw new IllegalArgumentException("Storage-engine node local path cannot leave the automation workspace"); + } + rejectSymbolicLinks(workspace, resolved); + return resolved.toString(); + } + + /** + * Rejects existing symbolic links along a path before it is handed to a storage engine. + * + * @param workspace project-owned automation runtime directory + * @param resolved normalized path inside the workspace + */ + private static void rejectSymbolicLinks(Path workspace, Path resolved) { + Path current = workspace; + for (Path segment : workspace.relativize(resolved)) { + current = current.resolve(segment); + if (Files.exists(current) && Files.isSymbolicLink(current)) { + throw new IllegalArgumentException("Storage-engine node local path cannot contain symbolic links"); + } + } + } + } diff --git a/src/prerna/reactor/automation/utils/PixelExecutionUtils.java b/src/prerna/reactor/automation/utils/PixelExecutionUtils.java index b4a192a5721..2a231104379 100644 --- a/src/prerna/reactor/automation/utils/PixelExecutionUtils.java +++ b/src/prerna/reactor/automation/utils/PixelExecutionUtils.java @@ -31,18 +31,22 @@ import java.util.List; import java.util.Map; import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import prerna.om.Insight; import prerna.om.ThreadStore; +import prerna.sablecc2.PixelRunner; import prerna.sablecc2.om.PixelOperationType; import prerna.sablecc2.om.nounmeta.NounMetadata; import prerna.sablecc2.om.task.ITask; @@ -54,7 +58,8 @@ *

    Handles the common pitfalls of raw {@code insight.runPixel()} calls: *

      *
    • {@link ITask} materialization - query reactors return lazy cursors that must be collected
      • - *
    • Timeout enforcement - prevents hung queries from blocking pipelines indefinitely
      • + *
    • Timeout enforcement - requests cancellation and prevents concurrent runs until the + * timed-out work has stopped
      • *
    • Error extraction - detects {@link prerna.sablecc2.om.PixelOperationType#ERROR} in results
      • *
    • Null-safe return - always returns a usable value
      • *
    @@ -106,7 +111,6 @@ public static Object runAndCollect(Insight insight, String pixel) { // -- Private implementation ---------------------------------------------------- private static NounMetadata executeWithTimeout(Insight insight, String pixel, int timeoutSeconds) { - // A new executor is created per timed call and shut down immediately after — no leak. ExecutorService executor = Executors.newSingleThreadExecutor(r -> { Thread t = new Thread(r, "automation-pixel-exec"); t.setDaemon(true); @@ -118,6 +122,9 @@ private static NounMetadata executeWithTimeout(Insight insight, String pixel, in Map callerContext = ThreadStore.getTheadMapObject(); final Map contextSnapshot = callerContext != null ? new HashMap<>(callerContext) : null; + AtomicReference activeRunner = new AtomicReference<>(); + AtomicBoolean timeoutRequested = new AtomicBoolean(false); + CountDownLatch executionTerminated = new CountDownLatch(1); try { Callable task = () -> { @@ -130,9 +137,15 @@ private static NounMetadata executeWithTimeout(Insight insight, String pixel, in ThreadStore.setThreadMapObject(contextSnapshot); } try { - return executeDirectly(insight, pixel); + PixelRunner runner = insight.getPixelRunner(); + activeRunner.set(runner); + if (timeoutRequested.get()) { + runner.cancelRequest(); + } + return executeDirectly(insight, runner, pixel); } finally { ThreadStore.remove(); + executionTerminated.countDown(); } }; @@ -140,7 +153,18 @@ private static NounMetadata executeWithTimeout(Insight insight, String pixel, in try { return future.get(timeoutSeconds, TimeUnit.SECONDS); } catch (TimeoutException e) { + timeoutRequested.set(true); + PixelRunner runner = activeRunner.get(); + if (runner != null) { + runner.cancelRequest(); + } future.cancel(true); + + // Future.cancel(true) only interrupts the worker. Some engine calls do not honor + // interruption immediately, so returning here would let AutomationRunEngine release + // its project lease while side effects may still be in flight. Hold the caller until + // the worker has actually terminated, then report the timeout. + awaitTermination(executionTerminated); throw new AutomationNodeTimeoutException(pixel, timeoutSeconds); } catch (ExecutionException e) { Throwable cause = e.getCause(); @@ -151,17 +175,36 @@ private static NounMetadata executeWithTimeout(Insight insight, String pixel, in throw new IllegalStateException("Pixel execution interrupted", e); } } finally { - executor.shutdownNow(); + executor.shutdown(); } } private static NounMetadata executeDirectly(Insight insight, String pixel) { - List results = insight.runPixel(pixel).getResults(); + return executeDirectly(insight, insight.getPixelRunner(), pixel); + } + + private static NounMetadata executeDirectly(Insight insight, PixelRunner runner, String pixel) { + List results = insight.runPixel(runner, pixel).getResults(); if (results == null || results.isEmpty()) return null; // For multi-statement pixels the meaningful result is the last one. return results.get(results.size() - 1); } + private static void awaitTermination(CountDownLatch executionTerminated) { + boolean interrupted = false; + while (true) { + try { + executionTerminated.await(); + break; + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + private static void checkForError(NounMetadata result, String pixel) { if (result.getOpType() != null && result.getOpType().contains(PixelOperationType.ERROR)) { String errorMsg = result.getValue() != null ? result.getValue().toString() : "Unknown pixel error"; diff --git a/src/prerna/reactor/project/CreateProjectReactor.java b/src/prerna/reactor/project/CreateProjectReactor.java index c489ea1db79..eaa124bd402 100644 --- a/src/prerna/reactor/project/CreateProjectReactor.java +++ b/src/prerna/reactor/project/CreateProjectReactor.java @@ -131,6 +131,11 @@ public NounMetadata execute() { + "Use CreateNotebook(project='...') instead — it scaffolds the sample .ipynb " + "file that CreateProject skips."); } + if (projectType == IProject.PROJECT_TYPE.AUTOMATION) { + throw new IllegalArgumentException("CreateProject cannot create AUTOMATION-type projects. " + + "Use CreateAutomation(projectName='...') instead — it scaffolds the automation " + + "definition, configuration, and MCP tool metadata."); + } } String gitProvider = this.keyValue.get(this.keysToGet[index++]); String gitCloneUrl = this.keyValue.get(this.keysToGet[index++]); From 581b31ed270fa31de2e2e69f5528aa913008ad02 Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Thu, 13 Aug 2026 15:48:09 -0400 Subject: [PATCH 23/25] feat: add new db colms for graph nodes --- .../automation/AutomationConstants.java | 3 + .../automation/AutomationDatabaseUtility.java | 74 +++-- .../AutomationDefinitionValidator.java | 285 ++++++++++++++++++ .../automation/AutomationOwlCreator.java | 6 + .../automation/SaveAutomationReactor.java | 1 + .../automation/TriggerAutomationReactor.java | 19 +- .../AutomationDefinitionValidatorTest.java | 93 ++++++ 7 files changed, 443 insertions(+), 38 deletions(-) create mode 100644 src/prerna/reactor/automation/AutomationDefinitionValidator.java create mode 100644 test/prerna/reactor/automation/AutomationDefinitionValidatorTest.java diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java index 7b3284892c7..850c1145551 100644 --- a/src/prerna/reactor/automation/AutomationConstants.java +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -49,6 +49,9 @@ private AutomationConstants() {} public static final String RUN_ID = "RUN_ID"; public static final String PROJECT_ID = "PROJECT_ID"; public static final String AUTOMATION_ID = "AUTOMATION_ID"; + public static final String DEFINITION_VERSION = "DEFINITION_VERSION"; + public static final String DEFINITION_HASH = "DEFINITION_HASH"; + public static final String DEFINITION_SNAPSHOT = "DEFINITION_SNAPSHOT"; public static final String STATUS = "STATUS"; public static final String TRIGGER_TYPE = "TRIGGER_TYPE"; public static final String STARTED_AT = "STARTED_AT"; diff --git a/src/prerna/reactor/automation/AutomationDatabaseUtility.java b/src/prerna/reactor/automation/AutomationDatabaseUtility.java index 781ad6cb3db..5b68aa26cc5 100644 --- a/src/prerna/reactor/automation/AutomationDatabaseUtility.java +++ b/src/prerna/reactor/automation/AutomationDatabaseUtility.java @@ -37,6 +37,9 @@ import static prerna.reactor.automation.AutomationConstants.COMPLETED_AT; import static prerna.reactor.automation.AutomationConstants.COMPLETED_NODES; import static prerna.reactor.automation.AutomationConstants.CREATED_BY; +import static prerna.reactor.automation.AutomationConstants.DEFINITION_HASH; +import static prerna.reactor.automation.AutomationConstants.DEFINITION_SNAPSHOT; +import static prerna.reactor.automation.AutomationConstants.DEFINITION_VERSION; import static prerna.reactor.automation.AutomationConstants.DURATION_MS; import static prerna.reactor.automation.AutomationConstants.ERROR_MESSAGE; import static prerna.reactor.automation.AutomationConstants.EXECUTION_ORDER; @@ -79,6 +82,7 @@ import static prerna.reactor.automation.AutomationConstants.VARCHAR_50; import static prerna.reactor.automation.AutomationConstants.VARCHAR_500; +import java.io.UnsupportedEncodingException; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -134,9 +138,10 @@ private AutomationDatabaseUtility() { // AUTOMATION_RUNS private static final String INSERT_RUN = """ INSERT INTO AUTOMATION_RUNS \ - (RUN_ID, PROJECT_ID, AUTOMATION_ID, STATUS, TRIGGER_TYPE, \ + (RUN_ID, PROJECT_ID, AUTOMATION_ID, DEFINITION_VERSION, DEFINITION_HASH, DEFINITION_SNAPSHOT, \ + STATUS, TRIGGER_TYPE, \ STARTED_AT, LAST_HEARTBEAT, TOTAL_NODES, COMPLETED_NODES, CREATED_BY) \ - VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?)"""; + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?)"""; private static final String UPDATE_RUN_STATUS = """ UPDATE AUTOMATION_RUNS SET STATUS = ?, COMPLETED_AT = ?, \ @@ -498,6 +503,7 @@ public static boolean isCancelRequested(String runId) { * Inserts a new automation run record. */ public static boolean insertRun(String runId, String projectId, String automationId, + int definitionVersion, String definitionHash, String definitionSnapshot, String triggerType, int totalNodes, String createdBy) { IRDBMSEngine schedulerDb = getSchedulerDb(); if (schedulerDb == null) return false; @@ -505,6 +511,7 @@ public static boolean insertRun(String runId, String projectId, String automatio Connection conn = null; try { conn = schedulerDb.getConnection(); + AbstractSqlQueryUtil queryUtil = schedulerDb.getQueryUtil(); Timestamp now = toTimestamp(Instant.now()); try (PreparedStatement ps = conn.prepareStatement(INSERT_RUN)) { @@ -512,6 +519,9 @@ public static boolean insertRun(String runId, String projectId, String automatio ps.setString(index++, runId); ps.setString(index++, projectId); ps.setString(index++, automationId); + ps.setInt(index++, definitionVersion); + ps.setString(index++, definitionHash); + queryUtil.handleInsertionOfClob(conn, ps, definitionSnapshot, index++, AutomationExecutionUtils.GSON); ps.setString(index++, STATUS_RUNNING); ps.setString(index++, triggerType); ps.setTimestamp(index++, now); @@ -525,7 +535,7 @@ public static boolean insertRun(String runId, String projectId, String automatio conn.commit(); } return true; - } catch (SQLException e) { + } catch (SQLException | UnsupportedEncodingException e) { classLogger.error("Failed to insert automation run '{}'", runId, e); return false; } finally { @@ -666,6 +676,9 @@ public static List> getRunsForProject(String projectId, int qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + RUN_ID, RUN_ID)); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + PROJECT_ID, PROJECT_ID)); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + AUTOMATION_ID, AUTOMATION_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + DEFINITION_VERSION, DEFINITION_VERSION)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + DEFINITION_HASH, DEFINITION_HASH)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + DEFINITION_SNAPSHOT, DEFINITION_SNAPSHOT)); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + STATUS, STATUS)); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + TRIGGER_TYPE, TRIGGER_TYPE)); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + STARTED_AT, STARTED_AT)); @@ -697,6 +710,9 @@ public static Map getRunDetail(String runId) { qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + RUN_ID, RUN_ID)); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + PROJECT_ID, PROJECT_ID)); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + AUTOMATION_ID, AUTOMATION_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + DEFINITION_VERSION, DEFINITION_VERSION)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + DEFINITION_HASH, DEFINITION_HASH)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + DEFINITION_SNAPSHOT, DEFINITION_SNAPSHOT)); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + STATUS, STATUS)); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + TRIGGER_TYPE, TRIGGER_TYPE)); qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + STARTED_AT, STARTED_AT)); @@ -935,35 +951,39 @@ private static void createAutomationRunsTable(Connection conn, AbstractSqlQueryU String tableName = TABLE_AUTOMATION_RUNS; - if (!allowIfExists && queryUtil.tableExists(conn, tableName, database, schema)) { - return; - } + boolean tableExists = !allowIfExists && queryUtil.tableExists(conn, tableName, database, schema); + if (!tableExists) { + String[] colNames = { RUN_ID, PROJECT_ID, AUTOMATION_ID, DEFINITION_VERSION, DEFINITION_HASH, + DEFINITION_SNAPSHOT, STATUS, TRIGGER_TYPE, STARTED_AT, COMPLETED_AT, FAILED_NODE_ID, + ERROR_MESSAGE, LAST_HEARTBEAT, TOTAL_NODES, COMPLETED_NODES, CREATED_BY, + CANCEL_REQUESTED, RESULT_SUMMARY_COL }; + String[] types = { VARCHAR_255, VARCHAR_255, VARCHAR_255, INTEGER, VARCHAR_255, + clobType, VARCHAR_50, VARCHAR_50, dateTimeType, dateTimeType, VARCHAR_255, + clobType, dateTimeType, INTEGER, INTEGER, VARCHAR_255, + queryUtil.getBooleanDataTypeName(), VARCHAR_2000 }; + String[] constraints = { NOT_NULL, NOT_NULL, null, null, null, + null, NOT_NULL, NOT_NULL, NOT_NULL, null, null, + null, null, null, null, null, + null, null }; - String[] colNames = { RUN_ID, PROJECT_ID, AUTOMATION_ID, STATUS, TRIGGER_TYPE, - STARTED_AT, COMPLETED_AT, FAILED_NODE_ID, ERROR_MESSAGE, LAST_HEARTBEAT, - TOTAL_NODES, COMPLETED_NODES, CREATED_BY, CANCEL_REQUESTED, RESULT_SUMMARY_COL }; - String[] types = { VARCHAR_255, VARCHAR_255, VARCHAR_255, VARCHAR_50, VARCHAR_50, - dateTimeType, dateTimeType, VARCHAR_255, clobType, dateTimeType, - INTEGER, INTEGER, VARCHAR_255, queryUtil.getBooleanDataTypeName(), VARCHAR_2000 }; - String[] constraints = { NOT_NULL, NOT_NULL, null, NOT_NULL, NOT_NULL, - NOT_NULL, null, null, null, null, - null, null, null, null, null }; - - String sql; - if (allowIfExists) { - sql = queryUtil.createTableIfNotExistsWithCustomConstraints(tableName, colNames, types, constraints); - } else { - sql = queryUtil.createTableWithCustomConstraints(tableName, colNames, types, constraints); - } - classLogger.info("Creating table {}: {}", tableName, sql); - try (PreparedStatement ps = conn.prepareStatement(sql)) { - ps.execute(); + String sql; + if (allowIfExists) { + sql = queryUtil.createTableIfNotExistsWithCustomConstraints(tableName, colNames, types, constraints); + } else { + sql = queryUtil.createTableWithCustomConstraints(tableName, colNames, types, constraints); + } + classLogger.info("Creating table {}: {}", tableName, sql); + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } } - // Migrate installs that predate cluster-safe cancel + // Additive migration for existing installations. addColumnIfNotExists(conn, queryUtil, tableName, CANCEL_REQUESTED, queryUtil.getBooleanDataTypeName()); - // Migrate installs that predate result summary addColumnIfNotExists(conn, queryUtil, tableName, RESULT_SUMMARY_COL, VARCHAR_2000); + addColumnIfNotExists(conn, queryUtil, tableName, DEFINITION_VERSION, INTEGER); + addColumnIfNotExists(conn, queryUtil, tableName, DEFINITION_HASH, VARCHAR_255); + addColumnIfNotExists(conn, queryUtil, tableName, DEFINITION_SNAPSHOT, clobType); // Primary key addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, PK_AUTOMATION_RUNS, diff --git a/src/prerna/reactor/automation/AutomationDefinitionValidator.java b/src/prerna/reactor/automation/AutomationDefinitionValidator.java new file mode 100644 index 00000000000..53fbb992aab --- /dev/null +++ b/src/prerna/reactor/automation/AutomationDefinitionValidator.java @@ -0,0 +1,285 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import com.google.gson.JsonParseException; + +import prerna.reactor.automation.nodes.IAutomationNodeExecutor; +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +/** + * Validates automation definition documents and produces a stable run provenance snapshot. + */ +public final class AutomationDefinitionValidator { + + private AutomationDefinitionValidator() {} + + /** + * Parses and validates an automation JSON document. + * + * @param json automation definition JSON + * @return validated document metadata + */ + public static ValidatedDefinition parseAndValidate(String json) { + if (json == null || json.isBlank()) { + throw new IllegalArgumentException("Automation definition must be a nonblank JSON object."); + } + try { + Map document = AutomationExecutionUtils.GSON.fromJson(json, AutomationExecutionUtils.MAP_TYPE); + return validate(document); + } catch (JsonParseException e) { + throw new IllegalArgumentException("Automation definition must be valid JSON.", e); + } + } + + /** + * Validates a parsed automation document. The returned nodes are the document's node maps, + * allowing permitted runtime overrides to be revalidated and snapshotted without reparsing. + * + * @param document parsed automation definition + * @return validated document metadata + */ + public static ValidatedDefinition validate(Map document) { + if (document == null) { + throw new IllegalArgumentException("Automation definition must be a JSON object."); + } + + int version = validateVersion(document.get(AutomationConstants.DOC_VERSION)); + Map graph = requireMap(document.get(AutomationConstants.DOC_GRAPH), "graph"); + List> nodes = requireMapList(graph.get(AutomationConstants.DOC_NODES), "graph.nodes"); + List> edges = requireMapList(graph.get(AutomationConstants.DOC_EDGES), "graph.edges"); + + Set nodeIds = validateNodes(nodes); + validateEdgesAndDag(edges, nodeIds); + + String snapshot = toCanonicalJson(document); + return new ValidatedDefinition(document, nodes, version, snapshot, sha256(snapshot)); + } + + private static int validateVersion(Object value) { + if (!(value instanceof Number)) { + throw new IllegalArgumentException("Automation definition version must be " + AutomationConstants.DOC_CURRENT_VERSION + "."); + } + double number = ((Number) value).doubleValue(); + if (!Double.isFinite(number) || number != Math.rint(number) + || number != AutomationConstants.DOC_CURRENT_VERSION) { + throw new IllegalArgumentException("Unsupported automation definition version: " + value + "."); + } + return (int) number; + } + + private static Set validateNodes(List> nodes) { + Set nodeIds = new HashSet<>(); + int triggerCount = 0; + for (int i = 0; i < nodes.size(); i++) { + Map node = nodes.get(i); + String nodeId = requireNonblankString(node.get(AutomationConstants.NODE_FIELD_ID), + "graph.nodes[" + i + "].id"); + if (!nodeIds.add(nodeId)) { + throw new IllegalArgumentException("Automation definition has duplicate node id: " + nodeId + "."); + } + + String type = requireNonblankString(node.get(AutomationConstants.NODE_FIELD_TYPE), + "graph.nodes[" + i + "].type"); + if (AutomationConstants.NODE_TRIGGER.equals(type)) { + triggerCount++; + } else if (!IAutomationNodeExecutor.EXECUTORS.containsKey(type)) { + throw new IllegalArgumentException("Unsupported automation node type: " + type + "."); + } + } + if (triggerCount != 1) { + throw new IllegalArgumentException("Automation definition must contain exactly one trigger node."); + } + return nodeIds; + } + + private static void validateEdgesAndDag(List> edges, Set nodeIds) { + Map> adjacency = new HashMap<>(); + Map indegrees = new HashMap<>(); + for (String nodeId : nodeIds) { + adjacency.put(nodeId, new ArrayList<>()); + indegrees.put(nodeId, 0); + } + + for (int i = 0; i < edges.size(); i++) { + Map edge = edges.get(i); + String source = requireNonblankString(edge.get(AutomationConstants.EDGE_FIELD_SOURCE), + "graph.edges[" + i + "].source"); + String target = requireNonblankString(edge.get(AutomationConstants.EDGE_FIELD_TARGET), + "graph.edges[" + i + "].target"); + if (!nodeIds.contains(source) || !nodeIds.contains(target)) { + throw new IllegalArgumentException("Automation edge references an unknown node: " + source + " -> " + target + "."); + } + if (source.equals(target)) { + throw new IllegalArgumentException("Automation edge cannot reference the same source and target node: " + source + "."); + } + adjacency.get(source).add(target); + indegrees.put(target, indegrees.get(target) + 1); + } + + ArrayDeque ready = new ArrayDeque<>(); + for (Map.Entry entry : indegrees.entrySet()) { + if (entry.getValue() == 0) { + ready.add(entry.getKey()); + } + } + int visited = 0; + while (!ready.isEmpty()) { + String nodeId = ready.remove(); + visited++; + for (String target : adjacency.get(nodeId)) { + int remaining = indegrees.get(target) - 1; + indegrees.put(target, remaining); + if (remaining == 0) { + ready.add(target); + } + } + } + if (visited != nodeIds.size()) { + throw new IllegalArgumentException("Automation definition graph must be acyclic."); + } + } + + private static Map requireMap(Object value, String field) { + if (!(value instanceof Map)) { + throw new IllegalArgumentException("Automation definition field '" + field + "' must be an object."); + } + for (Map.Entry entry : ((Map) value).entrySet()) { + if (!(entry.getKey() instanceof String)) { + throw new IllegalArgumentException("Automation definition field '" + field + "' has a non-string key."); + } + } + @SuppressWarnings("unchecked") + Map source = (Map) value; + return source; + } + + private static List> requireMapList(Object value, String field) { + if (!(value instanceof List)) { + throw new IllegalArgumentException("Automation definition field '" + field + "' must be an array."); + } + List> maps = new ArrayList<>(); + List values = (List) value; + for (int i = 0; i < values.size(); i++) { + maps.add(requireMap(values.get(i), field + "[" + i + "]")); + } + return maps; + } + + private static String requireNonblankString(Object value, String field) { + if (!(value instanceof String) || ((String) value).isBlank()) { + throw new IllegalArgumentException("Automation definition field '" + field + "' must be a nonblank string."); + } + return (String) value; + } + + private static String toCanonicalJson(Map document) { + return AutomationExecutionUtils.GSON.toJson(canonicalize(document)); + } + + private static Object canonicalize(Object value) { + if (value instanceof Map) { + Map sorted = new TreeMap<>(); + for (Map.Entry entry : ((Map) value).entrySet()) { + if (!(entry.getKey() instanceof String)) { + throw new IllegalArgumentException("Automation definition contains a non-string object key."); + } + sorted.put((String) entry.getKey(), canonicalize(entry.getValue())); + } + return sorted; + } + if (value instanceof List) { + List values = new ArrayList<>(); + for (Object element : (List) value) { + values.add(canonicalize(element)); + } + return values; + } + return value; + } + + private static String sha256(String value) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); + return java.util.HexFormat.of().formatHex(hash); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable.", e); + } + } + + /** + * Immutable metadata for a validated document and its canonical execution snapshot. + */ + public static final class ValidatedDefinition { + + private final Map document; + private final List> nodes; + private final int version; + private final String snapshot; + private final String hash; + + private ValidatedDefinition(Map document, List> nodes, + int version, String snapshot, String hash) { + this.document = document; + this.nodes = nodes; + this.version = version; + this.snapshot = snapshot; + this.hash = hash; + } + + public Map getDocument() { + return document; + } + + public List> getNodes() { + return nodes; + } + + public int getVersion() { + return version; + } + + public String getSnapshot() { + return snapshot; + } + + public String getHash() { + return hash; + } + } +} diff --git a/src/prerna/reactor/automation/AutomationOwlCreator.java b/src/prerna/reactor/automation/AutomationOwlCreator.java index 00b7cfa02a1..740e009b5f0 100644 --- a/src/prerna/reactor/automation/AutomationOwlCreator.java +++ b/src/prerna/reactor/automation/AutomationOwlCreator.java @@ -34,6 +34,9 @@ import static prerna.reactor.automation.AutomationConstants.COMPLETED_AT; import static prerna.reactor.automation.AutomationConstants.COMPLETED_NODES; import static prerna.reactor.automation.AutomationConstants.CREATED_BY; +import static prerna.reactor.automation.AutomationConstants.DEFINITION_HASH; +import static prerna.reactor.automation.AutomationConstants.DEFINITION_SNAPSHOT; +import static prerna.reactor.automation.AutomationConstants.DEFINITION_VERSION; import static prerna.reactor.automation.AutomationConstants.DURATION_MS; import static prerna.reactor.automation.AutomationConstants.ERROR_MESSAGE; import static prerna.reactor.automation.AutomationConstants.EXECUTION_ORDER; @@ -95,6 +98,9 @@ public void createColumnsAndTypes() { Pair.with(RUN_ID, VARCHAR_255), Pair.with(PROJECT_ID, VARCHAR_255), Pair.with(AUTOMATION_ID, VARCHAR_255), + Pair.with(DEFINITION_VERSION, INTEGER), + Pair.with(DEFINITION_HASH, VARCHAR_255), + Pair.with(DEFINITION_SNAPSHOT, CLOB), Pair.with(STATUS, VARCHAR_255), Pair.with(TRIGGER_TYPE, VARCHAR_255), Pair.with(STARTED_AT, TIMESTAMP), diff --git a/src/prerna/reactor/automation/SaveAutomationReactor.java b/src/prerna/reactor/automation/SaveAutomationReactor.java index 329f095a576..792b16617e4 100644 --- a/src/prerna/reactor/automation/SaveAutomationReactor.java +++ b/src/prerna/reactor/automation/SaveAutomationReactor.java @@ -86,6 +86,7 @@ public NounMetadata execute() { } catch (Exception e) { json = jsonEncoded; } + AutomationDefinitionValidator.parseAndValidate(json); IProject project = Utility.getProject(projectId); String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); diff --git a/src/prerna/reactor/automation/TriggerAutomationReactor.java b/src/prerna/reactor/automation/TriggerAutomationReactor.java index 741d7df0791..e18abde26da 100644 --- a/src/prerna/reactor/automation/TriggerAutomationReactor.java +++ b/src/prerna/reactor/automation/TriggerAutomationReactor.java @@ -29,7 +29,6 @@ import prerna.reactor.automation.utils.AutomationExecutionUtils; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -73,15 +72,9 @@ public NounMetadata execute() { // Validate the automation has runnable steps BEFORE claiming the run slot, // so a bad automation never leaves a stale active-run record. - Map doc = AutomationExecutionUtils.loadAutomationDoc(projectId); - @SuppressWarnings("unchecked") - Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); - @SuppressWarnings("unchecked") - List> nodes = (List>) graph.get(AutomationConstants.DOC_NODES); - List> ordered = nodes != null ? nodes : new ArrayList<>(); - if (ordered.isEmpty()) { - throw new IllegalArgumentException("Automation has no nodes to execute"); - } + AutomationDefinitionValidator.ValidatedDefinition definition = + AutomationDefinitionValidator.validate(AutomationExecutionUtils.loadAutomationDoc(projectId)); + List> ordered = definition.getNodes(); long nonTriggerCount = ordered.stream() .filter(n -> !AutomationConstants.NODE_TRIGGER.equals(n.get(AutomationConstants.NODE_FIELD_TYPE))) .count(); @@ -103,6 +96,8 @@ public NounMetadata execute() { @SuppressWarnings("unchecked") Map inputsMap = this.getMap(AutomationConstants.AUTOMATION_INPUTS_KEY); AutomationExecutionUtils.applyPlaygroundInputs(ordered, inputsMap); + definition = AutomationDefinitionValidator.validate(definition.getDocument()); + ordered = definition.getNodes(); String triggerType = this.keyValue.get(AutomationConstants.AUTOMATION_TRIGGER_TYPE_KEY); if (triggerType == null || triggerType.isBlank()) { @@ -113,6 +108,7 @@ public NounMetadata execute() { triggerType = AutomationConstants.TRIGGER_MANUAL; } AutomationDatabaseUtility.insertRun(runId, projectId, AutomationConstants.DEFAULT_AUTOMATION_ID, + definition.getVersion(), definition.getHash(), definition.getSnapshot(), triggerType, ordered.size(), userId); AutomationDatabaseUtility.insertAllNodeOutputs(runId, ordered); @@ -147,7 +143,8 @@ public NounMetadata execute() { boolean runSucceeded = AutomationConstants.STATUS_SUCCESS.equals(runDetail.get(AutomationConstants.STATUS)); // Short summary shown in the sidebar UI. String summary = runSucceeded - ? AutomationExecutionUtils.buildSummaryMessage(doc, finalScope, configMap, completedCount, ordered.size()) + ? AutomationExecutionUtils.buildSummaryMessage(definition.getDocument(), finalScope, configMap, + completedCount, ordered.size()) : buildFailureSummary(runDetail); runDetail.put(AutomationConstants.RESULT_SUMMARY, summary); AutomationDatabaseUtility.updateRunSummary(runId, summary); diff --git a/test/prerna/reactor/automation/AutomationDefinitionValidatorTest.java b/test/prerna/reactor/automation/AutomationDefinitionValidatorTest.java new file mode 100644 index 00000000000..b7f11ed1137 --- /dev/null +++ b/test/prerna/reactor/automation/AutomationDefinitionValidatorTest.java @@ -0,0 +1,93 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +class AutomationDefinitionValidatorTest { + + @Test + void validatesTriggerOnlyDefinitionWithoutRequiringRunnableSteps() { + AutomationDefinitionValidator.ValidatedDefinition definition = + AutomationDefinitionValidator.parseAndValidate(document( + "[{\"id\":\"trigger\",\"type\":\"trigger\"}]", "[]")); + + assertEquals(AutomationConstants.DOC_CURRENT_VERSION, definition.getVersion()); + assertEquals(1, definition.getNodes().size()); + } + + @Test + void canonicalizesEquivalentDocumentsToTheSameSnapshotAndHash() { + String first = """ + {"version":1,"graph":{"nodes":[{"id":"trigger","type":"trigger"},{"id":"wait","type":"wait"}], + "edges":[{"source":"trigger","target":"wait"}]}}"""; + String second = """ + {"graph":{"edges":[{"target":"wait","source":"trigger"}], + "nodes":[{"type":"trigger","id":"trigger"},{"type":"wait","id":"wait"}]},"version":1}"""; + + AutomationDefinitionValidator.ValidatedDefinition firstDefinition = + AutomationDefinitionValidator.parseAndValidate(first); + AutomationDefinitionValidator.ValidatedDefinition secondDefinition = + AutomationDefinitionValidator.parseAndValidate(second); + + assertEquals(firstDefinition.getSnapshot(), secondDefinition.getSnapshot()); + assertEquals(firstDefinition.getHash(), secondDefinition.getHash()); + } + + @Test + void rejectsInvalidVersionsAndNodes() { + assertThrows(IllegalArgumentException.class, () -> + AutomationDefinitionValidator.parseAndValidate( + "{\"version\":2,\"graph\":{\"nodes\":[],\"edges\":[]}}")); + assertThrows(IllegalArgumentException.class, () -> + AutomationDefinitionValidator.parseAndValidate(document( + "[{\"id\":\"trigger\",\"type\":\"trigger\"},{\"id\":\"trigger\",\"type\":\"wait\"}]", "[]"))); + assertThrows(IllegalArgumentException.class, () -> + AutomationDefinitionValidator.parseAndValidate(document( + "[{\"id\":\"trigger\",\"type\":\"trigger\"},{\"id\":\"unknown\",\"type\":\"unknown\"}]", "[]"))); + } + + @Test + void rejectsInvalidEdgesAndCycles() { + assertThrows(IllegalArgumentException.class, () -> + AutomationDefinitionValidator.parseAndValidate(document( + "[{\"id\":\"trigger\",\"type\":\"trigger\"}]", "[{\"source\":\"trigger\",\"target\":\"missing\"}]"))); + assertThrows(IllegalArgumentException.class, () -> + AutomationDefinitionValidator.parseAndValidate(document( + "[{\"id\":\"trigger\",\"type\":\"trigger\"}]", "[{\"source\":\"trigger\",\"target\":\"trigger\"}]"))); + assertThrows(IllegalArgumentException.class, () -> + AutomationDefinitionValidator.parseAndValidate(document( + "[{\"id\":\"trigger\",\"type\":\"trigger\"},{\"id\":\"one\",\"type\":\"wait\"},{\"id\":\"two\",\"type\":\"wait\"}]", + "[{\"source\":\"trigger\",\"target\":\"one\"},{\"source\":\"one\",\"target\":\"two\"},{\"source\":\"two\",\"target\":\"one\"}]"))); + } + + private static String document(String nodes, String edges) { + return "{\"version\":1,\"graph\":{\"nodes\":" + nodes + ",\"edges\":" + edges + "}}"; + } +} From 731c5e886b7b84f8351dc681cc5e3f73818d3c88 Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Thu, 13 Aug 2026 16:26:30 -0400 Subject: [PATCH 24/25] chore: fixing bugs i added --- .../automation/AutomationDatabaseUtility.java | 34 +++++++++++ .../AutomationDefinitionValidator.java | 59 ++++++++++++++++++- .../automation/AutomationRunEngine.java | 19 ++++-- .../automation/TriggerAutomationReactor.java | 11 +--- .../AutomationDefinitionValidatorTest.java | 19 ++++++ 5 files changed, 126 insertions(+), 16 deletions(-) diff --git a/src/prerna/reactor/automation/AutomationDatabaseUtility.java b/src/prerna/reactor/automation/AutomationDatabaseUtility.java index 5b68aa26cc5..054256bf9bb 100644 --- a/src/prerna/reactor/automation/AutomationDatabaseUtility.java +++ b/src/prerna/reactor/automation/AutomationDatabaseUtility.java @@ -57,6 +57,7 @@ import static prerna.reactor.automation.AutomationConstants.NODE_STATUS_FAILED; import static prerna.reactor.automation.AutomationConstants.NODE_STATUS_PENDING; import static prerna.reactor.automation.AutomationConstants.NODE_STATUS_RUNNING; +import static prerna.reactor.automation.AutomationConstants.NODE_STATUS_SKIPPED; import static prerna.reactor.automation.AutomationConstants.NODE_STATUS_SUCCESS; import static prerna.reactor.automation.AutomationConstants.NOT_NULL; import static prerna.reactor.automation.AutomationConstants.OUTPUT_PREVIEW; @@ -188,6 +189,9 @@ private AutomationDatabaseUtility() { private static final String UPDATE_NODE_STATUS = "UPDATE AUTOMATION_NODE_OUTPUTS SET STATUS = ?, STARTED_AT = ? WHERE RUN_ID = ? AND NODE_ID = ?"; + private static final String SKIP_PENDING_NODE_OUTPUTS = + "UPDATE AUTOMATION_NODE_OUTPUTS SET STATUS = ?, ERROR_MESSAGE = ? WHERE RUN_ID = ? AND STATUS = ?"; + // -- Initialization ------------------------------------------------------------ /** @@ -790,6 +794,7 @@ public static boolean markNodeRunning(String runId, String nodeId) { ps.setString(index++, nodeId); ps.executeUpdate(); } + if (!conn.getAutoCommit()) { conn.commit(); } @@ -803,6 +808,35 @@ public static boolean markNodeRunning(String runId, String nodeId) { } } + /** + * Marks all nodes that did not start because the run reached a terminal state as skipped. + */ + public static boolean skipPendingNodes(String runId, String reason) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(SKIP_PENDING_NODE_OUTPUTS)) { + ps.setString(1, NODE_STATUS_SKIPPED); + setNullableString(ps, 2, reason); + ps.setString(3, runId); + ps.setString(4, NODE_STATUS_PENDING); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to skip pending nodes for run '{}'", runId, e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + /** * Updates a node output after successful execution. */ diff --git a/src/prerna/reactor/automation/AutomationDefinitionValidator.java b/src/prerna/reactor/automation/AutomationDefinitionValidator.java index 53fbb992aab..cba1b1d3761 100644 --- a/src/prerna/reactor/automation/AutomationDefinitionValidator.java +++ b/src/prerna/reactor/automation/AutomationDefinitionValidator.java @@ -32,8 +32,10 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.PriorityQueue; import java.util.Set; import java.util.TreeMap; @@ -88,7 +90,7 @@ public static ValidatedDefinition validate(Map document) { validateEdgesAndDag(edges, nodeIds); String snapshot = toCanonicalJson(document); - return new ValidatedDefinition(document, nodes, version, snapshot, sha256(snapshot)); + return new ValidatedDefinition(document, nodes, edges, version, snapshot, sha256(snapshot)); } private static int validateVersion(Object value) { @@ -175,6 +177,50 @@ private static void validateEdgesAndDag(List> edges, Set> topologicallyOrderNodes(List> nodes, + List> edges) { + Map> nodesById = new LinkedHashMap<>(); + Map> adjacency = new LinkedHashMap<>(); + Map indegrees = new LinkedHashMap<>(); + Map nodeIndexes = new HashMap<>(); + for (int index = 0; index < nodes.size(); index++) { + Map node = nodes.get(index); + String nodeId = (String) node.get(AutomationConstants.NODE_FIELD_ID); + nodesById.put(nodeId, node); + adjacency.put(nodeId, new ArrayList<>()); + indegrees.put(nodeId, 0); + nodeIndexes.put(nodeId, index); + } + for (Map edge : edges) { + String source = (String) edge.get(AutomationConstants.EDGE_FIELD_SOURCE); + String target = (String) edge.get(AutomationConstants.EDGE_FIELD_TARGET); + adjacency.get(source).add(target); + indegrees.put(target, indegrees.get(target) + 1); + } + + PriorityQueue ready = new PriorityQueue<>((left, right) -> + Integer.compare(nodeIndexes.get(left), nodeIndexes.get(right))); + for (String nodeId : nodesById.keySet()) { + if (indegrees.get(nodeId) == 0) { + ready.add(nodeId); + } + } + + List> ordered = new ArrayList<>(); + while (!ready.isEmpty()) { + String nodeId = ready.remove(); + ordered.add(nodesById.get(nodeId)); + for (String target : adjacency.get(nodeId)) { + int remaining = indegrees.get(target) - 1; + indegrees.put(target, remaining); + if (remaining == 0) { + ready.add(target); + } + } + } + return ordered; + } + private static Map requireMap(Object value, String field) { if (!(value instanceof Map)) { throw new IllegalArgumentException("Automation definition field '" + field + "' must be an object."); @@ -249,14 +295,17 @@ public static final class ValidatedDefinition { private final Map document; private final List> nodes; + private final List> edges; private final int version; private final String snapshot; private final String hash; private ValidatedDefinition(Map document, List> nodes, + List> edges, int version, String snapshot, String hash) { this.document = document; this.nodes = nodes; + this.edges = edges; this.version = version; this.snapshot = snapshot; this.hash = hash; @@ -270,6 +319,14 @@ public List> getNodes() { return nodes; } + /** + * Returns nodes in dependency order. Nodes that become ready together retain their + * persisted document order, keeping shared-scope execution deterministic. + */ + public List> getExecutionOrder() { + return topologicallyOrderNodes(nodes, edges); + } + public int getVersion() { return version; } diff --git a/src/prerna/reactor/automation/AutomationRunEngine.java b/src/prerna/reactor/automation/AutomationRunEngine.java index 8056d024e21..863992c951e 100644 --- a/src/prerna/reactor/automation/AutomationRunEngine.java +++ b/src/prerna/reactor/automation/AutomationRunEngine.java @@ -56,7 +56,7 @@ /** * Executes an automation run synchronously. Called by {@link TriggerAutomationReactor} * on the virtual thread provided by the platform's {@code runPixelAsync} endpoint. - * Iterates nodes in order, dispatches each to its {@link IAutomationNodeExecutor}, + * Iterates the dependency-ordered nodes supplied by the validated definition, dispatches each to its {@link IAutomationNodeExecutor}, * and writes per-node status to the DB as it goes. */ public final class AutomationRunEngine { @@ -120,6 +120,7 @@ public static Map run(String runId, String projectId, if (cancelled.get() || AutomationDatabaseUtility.isCancelRequested(runId)) { classLogger.info("Automation run {} cancelled before node {} ({})", runId, nodeId, nodeLabel); + AutomationDatabaseUtility.skipPendingNodes(runId, "Run cancelled by user"); AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_CANCELLED, nodeId, "Run cancelled by user"); return scope; @@ -132,6 +133,7 @@ public static Map run(String runId, String projectId, nodeResult = executeSingleNode(runId, projectId, node, scope, configMap, cancelled, insight); } catch (PixelExecutionUtils.AutomationCancelledException ace) { classLogger.info("Automation run {} cancelled during node {} ({})", runId, nodeId, nodeLabel); + AutomationDatabaseUtility.skipPendingNodes(runId, "Run cancelled by user"); AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_CANCELLED, nodeId, ace.getMessage()); publishNodeEvent(jobId, nodeId, nodeLabel, AutomationConstants.STATUS_CANCELLED, null, null, @@ -155,6 +157,7 @@ public static Map run(String runId, String projectId, AutomationDatabaseUtility.updateHeartbeat(runId, completedCount); } else { classLogger.warn("Automation run {} failed at node {} ({}): {}", runId, nodeId, nodeLabel, errorMsg); + AutomationDatabaseUtility.skipPendingNodes(runId, "Skipped because an earlier node failed"); AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_FAILED, nodeId, errorMsg); return scope; @@ -219,17 +222,21 @@ private static Map executeSingleNode(String runId, String projec String outputVar = (String) node.get(AutomationConstants.NODE_FIELD_OUTPUT_VAR); String type = (String) node.get(AutomationConstants.NODE_FIELD_TYPE); - if (AutomationConstants.NODE_TRIGGER.equals(type)) { - return buildNodeResult(nodeId, nodeLabel, AutomationConstants.NODE_STATUS_SUCCESS, 0, - scope.get(AutomationConstants.SCOPE_TRIGGERED_AT), null); - } - classLogger.debug("Executing node {} ({}) type={} in run {}", nodeId, nodeLabel, type, runId); AutomationDatabaseUtility.markNodeRunning(runId, nodeId); Timestamp startedAt = Utility.getSqlTimestampUTC(java.time.LocalDateTime.ofInstant(Instant.now(), java.time.ZoneOffset.UTC)); long startMs = System.currentTimeMillis(); try { + if (AutomationConstants.NODE_TRIGGER.equals(type)) { + String triggeredAt = scope.get(AutomationConstants.SCOPE_TRIGGERED_AT); + AutomationDatabaseUtility.updateNodeSuccess(runId, nodeId, startedAt, 0, outputVar, triggeredAt, triggeredAt); + Map result = buildNodeResult(nodeId, nodeLabel, + AutomationConstants.NODE_STATUS_SUCCESS, 0, triggeredAt, null); + result.put(AutomationConstants.RESULT_OUTPUT_VALUE, triggeredAt); + return result; + } + AutomationNodeContext ctx = new AutomationNodeContext( runId, projectId, node, scope, configMap, insight, cancelFlag); diff --git a/src/prerna/reactor/automation/TriggerAutomationReactor.java b/src/prerna/reactor/automation/TriggerAutomationReactor.java index e18abde26da..3e7500e2ed3 100644 --- a/src/prerna/reactor/automation/TriggerAutomationReactor.java +++ b/src/prerna/reactor/automation/TriggerAutomationReactor.java @@ -74,7 +74,7 @@ public NounMetadata execute() { // so a bad automation never leaves a stale active-run record. AutomationDefinitionValidator.ValidatedDefinition definition = AutomationDefinitionValidator.validate(AutomationExecutionUtils.loadAutomationDoc(projectId)); - List> ordered = definition.getNodes(); + List> ordered = definition.getExecutionOrder(); long nonTriggerCount = ordered.stream() .filter(n -> !AutomationConstants.NODE_TRIGGER.equals(n.get(AutomationConstants.NODE_FIELD_TYPE))) .count(); @@ -97,7 +97,7 @@ public NounMetadata execute() { Map inputsMap = this.getMap(AutomationConstants.AUTOMATION_INPUTS_KEY); AutomationExecutionUtils.applyPlaygroundInputs(ordered, inputsMap); definition = AutomationDefinitionValidator.validate(definition.getDocument()); - ordered = definition.getNodes(); + ordered = definition.getExecutionOrder(); String triggerType = this.keyValue.get(AutomationConstants.AUTOMATION_TRIGGER_TYPE_KEY); if (triggerType == null || triggerType.isBlank()) { @@ -126,13 +126,6 @@ public NounMetadata execute() { completedCount++; } } - // Trigger nodes succeed immediately in the engine but never write a SUCCESS - // DB record, so add them back so the count reflects what the user sees. - int triggerCount = (int) ordered.stream() - .filter(n -> AutomationConstants.NODE_TRIGGER.equals(n.get(AutomationConstants.NODE_FIELD_TYPE))) - .count(); - completedCount += triggerCount; - if (runDetail == null) { runDetail = new HashMap<>(); runDetail.put(AutomationConstants.RUN_ID, runId); diff --git a/test/prerna/reactor/automation/AutomationDefinitionValidatorTest.java b/test/prerna/reactor/automation/AutomationDefinitionValidatorTest.java index b7f11ed1137..e0b680474cf 100644 --- a/test/prerna/reactor/automation/AutomationDefinitionValidatorTest.java +++ b/test/prerna/reactor/automation/AutomationDefinitionValidatorTest.java @@ -28,6 +28,9 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import java.util.List; +import java.util.Map; + import org.junit.jupiter.api.Test; class AutomationDefinitionValidatorTest { @@ -87,6 +90,22 @@ void rejectsInvalidEdgesAndCycles() { "[{\"source\":\"trigger\",\"target\":\"one\"},{\"source\":\"one\",\"target\":\"two\"},{\"source\":\"two\",\"target\":\"one\"}]"))); } + @Test + void ordersNodesByDependenciesWhilePreservingReadyNodeDocumentOrder() { + AutomationDefinitionValidator.ValidatedDefinition definition = + AutomationDefinitionValidator.parseAndValidate(document( + "[{\"id\":\"trigger\",\"type\":\"trigger\"},{\"id\":\"second\",\"type\":\"wait\"}," + + "{\"id\":\"first\",\"type\":\"wait\"},{\"id\":\"join\",\"type\":\"wait\"}]", + "[{\"source\":\"trigger\",\"target\":\"first\"},{\"source\":\"trigger\",\"target\":\"second\"}," + + "{\"source\":\"first\",\"target\":\"join\"},{\"source\":\"second\",\"target\":\"join\"}]")); + + List nodeIds = definition.getExecutionOrder().stream() + .map(node -> (String) node.get(AutomationConstants.NODE_FIELD_ID)) + .toList(); + + assertEquals(List.of("trigger", "second", "first", "join"), nodeIds); + } + private static String document(String nodes, String edges) { return "{\"version\":1,\"graph\":{\"nodes\":" + nodes + ",\"edges\":" + edges + "}}"; } From bf37265054cf4db6d6e8d4dc74a2dec5a81283b9 Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Fri, 14 Aug 2026 09:30:59 -0400 Subject: [PATCH 25/25] chore: small additions --- src/prerna/reactor/automation/AutomationRunEngine.java | 1 - src/prerna/reactor/automation/CreateAutomationReactor.java | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/src/prerna/reactor/automation/AutomationRunEngine.java b/src/prerna/reactor/automation/AutomationRunEngine.java index 863992c951e..5ad3efad169 100644 --- a/src/prerna/reactor/automation/AutomationRunEngine.java +++ b/src/prerna/reactor/automation/AutomationRunEngine.java @@ -239,7 +239,6 @@ private static Map executeSingleNode(String runId, String projec AutomationNodeContext ctx = new AutomationNodeContext( runId, projectId, node, scope, configMap, insight, cancelFlag); - IAutomationNodeExecutor executor = IAutomationNodeExecutor.EXECUTORS.get(type); if (executor == null) { throw new IllegalArgumentException("Unsupported node type: " + type); diff --git a/src/prerna/reactor/automation/CreateAutomationReactor.java b/src/prerna/reactor/automation/CreateAutomationReactor.java index 43e70997f71..36c5fd3fc62 100644 --- a/src/prerna/reactor/automation/CreateAutomationReactor.java +++ b/src/prerna/reactor/automation/CreateAutomationReactor.java @@ -116,6 +116,7 @@ public NounMetadata execute() { "SaveAutomation(project=[\"%s\"], json=[\"%s\"]);", projectId, encodedDefinition)); PixelExecutionUtils.runAndCollect(this.insight, String.format( "SaveAutomationConfig(project=[\"%s\"], config=[\"%s\"]);", projectId, encodedConfig)); + MCPUtility.addMCPTag(project); } catch (PixelExecutionUtils.AutomationPixelException e) { classLogger.error("Failed to scaffold automation project '{}'", projectName, e); throw new IllegalStateException(