diff --git a/src/prerna/project/api/IProject.java b/src/prerna/project/api/IProject.java index d8318b988bf..b44126153a9 100644 --- a/src/prerna/project/api/IProject.java +++ b/src/prerna/project/api/IProject.java @@ -60,7 +60,7 @@ public interface IProject extends IEngine, IMCP { String NOTEBOOK_FOLDER = ".notebooks"; enum PROJECT_TYPE { - BLOCKS, CODE, WORKSPACE, SKILL, INSIGHTS, NOTEBOOK, + BLOCKS, CODE, WORKSPACE, SKILL, INSIGHTS, NOTEBOOK, AUTOMATION, }; /** diff --git a/src/prerna/reactor/automation/AGENTS.md b/src/prerna/reactor/automation/AGENTS.md new file mode 100644 index 00000000000..f71f51d67f3 --- /dev/null +++ b/src/prerna/reactor/automation/AGENTS.md @@ -0,0 +1,140 @@ +# Automation Engine — Agent Guide + +Executes sequential node pipelines against SEMOSS engines. Users build a pipeline in the form editor (FE), save it as `automation.json`, then trigger it manually. Each run is tracked in the DB and the FE polls for progress. + +## How it works + +``` +FE calls runPixelAsync("TriggerAutomation(...)") + → Monolith spawns a virtual thread, returns jobId immediately + → FE polls GetActiveAutomationRun every 500ms (up to 10×) to get runId + +TriggerAutomationReactor (virtual thread, synchronous) + → reads automation.json (nodes in order) + → claims single-run slot (AUTOMATION_ACTIVE_RUN) → runId visible to FE + → inserts AUTOMATION_RUNS + AUTOMATION_NODE_OUTPUTS rows + → calls AutomationRunEngine.run() synchronously + → returns completed run result to jobId slot when done + +AutomationRunEngine (same virtual thread) + → iterates nodes in saved order + → dispatches each node to its IAutomationNodeExecutor + → writes node output + status to AUTOMATION_NODE_OUTPUTS after each node + → FE polls GetAutomationRun every 3s while runId is known +``` + +## Reactors + +| Reactor | Pixel | What it does | +| --- | --- | --- | +| `TriggerAutomationReactor` | `TriggerAutomation(project=["id"])` | Starts a run synchronously; returns completed run result | +| `GetActiveAutomationRunReactor` | `GetActiveAutomationRun(project=["id"])` | Returns `{RUN_ID, PROJECT_ID}` from active-run lock table; empty map when idle | +| `GetAutomationReactor` | `GetAutomation(project=["id"])` | Returns saved pipeline definition (automation.json) | +| `GetAutomationConfigReactor` | `GetAutomationConfig(project=["id"])` | Returns env var/secret config; masks sensitive values | +| `GetAutomationRunReactor` | `GetAutomationRun(project=["id"], runId=["id"])` | Returns live run state for FE polling | +| `ListAutomationRunsReactor` | `ListAutomationRuns(project=["id"])` | Returns run history | +| `CancelAutomationRunReactor` | `CancelAutomationRun(project=["id"], runId=["id"])` | Cancels an in-progress run | +| `SaveAutomationReactor` | `SaveAutomation(project=["id"], config=["{}"])` | Persists pipeline definition | +| `SaveAutomationConfigReactor` | `SaveAutomationConfig(project=["id"], config=["[]"])` | Persists env var config | +| `RunAutomationNodeReactor` | `RunAutomationNode(project=["id"], nodeId=["id"])` | Single-node test run; result not persisted | + +## Node types + +Each node type has a corresponding `IAutomationNodeExecutor` in `nodes/`: + +| Type | Executor | What it does | +| --- | --- | --- | +| `trigger` | (no executor) | Seed node; provides `triggered_at`, `date`, `run_id` scope vars | +| `database-engine` | `DatabaseEngineNodeExecutor` | Runs SQL via `SqlQuery` pixel | +| `model-engine` | `ModelEngineNodeExecutor` | LLM ask or embeddings via `IModelEngine` | +| `vector-engine` | `VectorEngineNodeExecutor` | Search, add, delete, list via `IVectorDatabaseEngine` | +| `storage-engine` | `StorageEngineNodeExecutor` | File operations via `IStorageEngine` | +| `function-engine` | `FunctionEngineNodeExecutor` | Function invocation via `IFunctionEngine` | +| `app` | `AppEngineNodeExecutor` | Arbitrary pixel, optionally scoped to a project | +| `wait` | `WaitNodeExecutor` | Sleep N seconds; cancel-aware | + +## DB tables + +| Table | Purpose | +| --- | --- | +| `AUTOMATION_ACTIVE_RUN` | PK on `PROJECT_ID` — enforces one concurrent run per project | +| `AUTOMATION_RUNS` | One row per run: status, timing, node counts | +| `AUTOMATION_NODE_OUTPUTS` | One row per node per run: status, output, preview, duration | + +## Key classes + +| Class | Purpose | +| --- | --- | +| `AutomationRunEngine` | Orchestrates a full pipeline run end-to-end | +| `AutomationExecutionUtils` | Shared statics: GSON, scope building, variable resolution, output transforms, preview generation | +| `AutomationGenerationUtils` | LLM/generation helpers: engine discovery, prompt building, response extraction | +| `AutomationDatabaseUtility` | All DB reads/writes for runs and node outputs | +| `PixelExecutionUtils` | Timeout-enforced pixel execution with ThreadStore propagation | +| `AutomationConstants` | String constants for all keys, statuses, and file names | + +--- + +## Adding a new node type + +1. **Create the executor** in `nodes/` implementing `IAutomationNodeExecutor`: + - Declare `private static final Logger classLogger = LogManager.getLogger(YourExecutor.class);` + - Log execution at `DEBUG` level with node label and key params before dispatching + - Use `AutomationExecutionUtils.resolve(value, scope, configMap)` for all `${var}` substitution + - Throw `IllegalArgumentException` for missing required config fields + - Use `AutomationExecutionUtils.GSON` — do not declare a local `Gson` instance + +2. **Register it** in the `EXECUTORS` map on `IAutomationNodeExecutor` (static field on the interface; use `Map.ofEntries` if adding an 11th entry — `Map.of` only supports 10) + +3. **Add the type constant** to `AutomationConstants` (e.g. `NODE_TYPE_FOO = "foo-engine"`) + +4. **Wire the FE** — add the node type to `automation.types.ts` and `automation.constants.ts` in `SemossWeb` + +## Logging rules + +Follow the platform standard — SLF4J `{}` placeholders, exception as the last argument: + +```java +// ✅ +classLogger.debug("Foo node \"{}\" executing operation={}", nodeLabel, operation); +classLogger.error("Foo node \"{}\" failed: {}", nodeLabel, e.getMessage(), e); + +// ❌ — never concatenate strings in log calls +classLogger.error("Foo node " + nodeLabel + " failed: " + e.getMessage()); +``` + +## Exception conventions + +```java +// Missing/invalid user input — use IllegalArgumentException +throw new IllegalArgumentException("Foo node \"" + nodeLabel + "\": 'engineId' is required"); + +// User-facing reactor errors — use SemossPixelException +throw new SemossPixelException("Project does not exist or user does not have access"); +``` + +## GSON + +Use the shared instance — never declare your own: + +```java +// ✅ +AutomationExecutionUtils.GSON.fromJson(json, AutomationExecutionUtils.MAP_TYPE); + +// ❌ +private static final Gson GSON = new GsonBuilder().create(); +``` + +## Reactor conventions + +- Keep reactors thin: parse params, auth-check, delegate, return. Business logic belongs in `AutomationRunEngine`, `AutomationExecutionUtils`, or an executor. +- Always call `organizeKeys()` before reading `this.keyValue` +- Use `SecurityProjectUtils.testUserProjectIdForAlias` to resolve alias → UUID before any lookup +- Add `getReactorDescription()` and `getDescriptionForKey()` to every reactor +- MCP-destructive reactors (save, trigger, cancel) must override `getMcpToolMetadata()` to `MCPExecution.ASK` + +## What not to change + +- `AutomationDatabaseUtility` — DB access is intentional; use `setNullableString`, `SelectQueryStruct`, try-with-resources +- `PixelExecutionUtils` — timeout + ThreadStore propagation; do not bypass +- `CancelAutomationRunReactor` — dual-signal cancel (DB flag + in-memory); both signals are required for cluster safety +- `claimActiveRun` — PK-violation is the concurrency guard; do not add a separate lock diff --git a/src/prerna/reactor/automation/AutomationAskRoomReactor.java b/src/prerna/reactor/automation/AutomationAskRoomReactor.java new file mode 100644 index 00000000000..d29ddddf7f0 --- /dev/null +++ b/src/prerna/reactor/automation/AutomationAskRoomReactor.java @@ -0,0 +1,202 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.AutomationGenerationUtils; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.engine.api.IModelEngine; +import prerna.engine.impl.model.RoomUtils; +import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.run.AgentRuntimeManager; +import prerna.reactor.agent.run.RunAgentRequest; +import prerna.reactor.agent.run.RunAgentResult; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Room-aware conversational AI assistant for building automation workflows. + * Uses the platform RunAgent harness for server-side history and MCP tool access. + * All user engines are registered as MCP tools so the model can query databases, + * search vectors, etc. during the design conversation. + * + *

Pixel: {@code AutomationAskRoom(project=["appId"], room=["roomId"], command=["base64text"])} + */ +public class AutomationAskRoomReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(AutomationAskRoomReactor.class); + + private static final String ROOM_KEY = "room"; + private static final String HARNESS_TYPE = "semoss"; + private static final int MAX_TURNS = 8; + private static final long CHAT_TIMEOUT_MS = 180_000L; + + private static final String SYSTEM_PROMPT = + "You are an AI assistant that helps users design automation workflows on a no-code platform. " + + "Workflows are linear sequences of steps: database queries, AI model calls, file storage, vector search, or custom functions. " + + "Only manual triggers exist - do not ask about scheduling.\n\n" + + "You have access to tools. Use them to help the user (for example, query a database to understand its structure " + + "so you can give accurate step descriptions).\n\n" + + "CONVERSATION PHASES:\n\n" + + "Phase 1 - Gather requirements (at most 1-2 short questions, one at a time):\n" + + "- Ask what the automation should do if unclear.\n" + + "- Ask where results should go, or one other essential clarification.\n" + + "- Keep responses under 50 words per question.\n\n" + + "Phase 2 - Plan + build signal (in ONE response, once you have enough info):\n" + + "- Present a concise numbered plain-English plan.\n" + + "- On the very next line after the plan, output the build signal JSON:\n" + + " {\"action\":\"build\",\"description\":\"\"}\n" + + "- The description must include all steps in enough detail for an AI to build them.\n" + + "- Do NOT ask 'does this look right?' - include the build signal in the same response as the plan.\n\n" + + "Phase 3 - If the user requests changes after seeing the plan:\n" + + "- Acknowledge in one short sentence.\n" + + "- Immediately output the revised plan + a new build signal in the same response.\n" + + "- Do NOT narrate what you will change - just show the revised plan and the signal.\n\n" + + "RULES: Never mention engine IDs, node types, or JSON structure to the user. Be concise."; + + public AutomationAskRoomReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), ROOM_KEY, ReactorKeysEnum.COMMAND.getKey() }; + this.keyRequired = new int[] { 1, 0, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } + + String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); + String roomId = this.keyValue.get(ROOM_KEY); + String rawCommand = this.keyValue.get(ReactorKeysEnum.COMMAND.getKey()); + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access."); + } + + String command = decodeCommand(rawCommand); + if (command == null || command.isBlank()) { + throw new IllegalArgumentException("command must not be empty."); + } + + String engineId = AutomationGenerationUtils.findFirstModelEngine(user); + if (engineId == null || engineId.isBlank()) { + throw new IllegalArgumentException( + "No AI model engine is available. Add a model engine connection to use this feature."); + } + + IModelEngine modelEngine = Utility.getModel(engineId); + if (modelEngine == null) { + throw new IllegalArgumentException("Model engine could not be loaded."); + } + + if (roomId == null || roomId.isBlank()) { + roomId = "automationchat" + projectId.replace("-", "").substring(0, Math.min(8, projectId.replace("-", "").length())); + } + + Map options = AutomationGenerationUtils.buildEngineMcpOptions(user, SYSTEM_PROMPT); + + RoomUtils.createRoomIfNotExists(roomId, this.insight, modelEngine, command, null, options, null, projectId, null); + + RunAgentRequest request = new RunAgentRequest( + roomId, command, engineId, HARNESS_TYPE, null, + MAX_TURNS, 0, null, null, null, null, this.insight); + + RunAgentResult handle = AgentRuntimeManager.get().run(request); + Map result; + try { + result = AgentRuntimeManager.get().waitForRun(handle.getRunId(), this.insight, CHAT_TIMEOUT_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Chat interrupted.", e); + } + + String status = (String) result.get("status"); + if ("FAILED".equals(status)) { + String errMsg = (String) result.get("errorMessage"); + classLogger.error("AutomationAskRoom run failed: project={} error={}", projectId, (errMsg != null ? errMsg : "unknown error")); + throw new RuntimeException("Chat failed: " + (errMsg != null ? errMsg : "unknown error")); + } + + String finalText = (String) result.get("finalText"); + if (finalText == null || finalText.isBlank()) { + throw new IllegalStateException("The AI model did not respond. Try again."); + } + + classLogger.info("AutomationAskRoom completed: project={}", projectId); + return new NounMetadata(finalText.strip(), PixelDataType.CONST_STRING, PixelOperationType.OPERATION); + } + + /** + * Decodes a base64-encoded command string. Falls back to the raw value when + * the input is not valid base64 (supports plain-text callers during testing). + */ + private static String decodeCommand(String raw) { + if (raw == null) { + return null; + } + try { + return new String(Base64.getDecoder().decode(raw.trim()), StandardCharsets.UTF_8); + } catch (Exception e) { + return raw; + } + } + + @Override + public String getReactorDescription() { + return "Room-aware conversational AI for designing automation workflows. " + + "Uses the platform RunAgent harness with MCP tool access - the model can query " + + "databases, search vectors, and use other engines during the conversation. " + + "History is managed server-side. " + + "Signals build-readiness via: {\"action\":\"build\",\"description\":\"...\"}"; + } + + @Override + protected String getDescriptionForKey(String key) { + return switch (key) { + case "project" -> "The project ID the automation belongs to."; + case "room" -> "Room ID for this conversation (defaults to automationchat{projectId})."; + case "command" -> "The user message, base64-encoded."; + default -> super.getDescriptionForKey(key); + }; + } +} diff --git a/src/prerna/reactor/automation/AutomationCancelledException.java b/src/prerna/reactor/automation/AutomationCancelledException.java new file mode 100644 index 00000000000..f01c68f361d --- /dev/null +++ b/src/prerna/reactor/automation/AutomationCancelledException.java @@ -0,0 +1,51 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.PixelExecutionUtils; + +/** + * Compatibility shim — extends {@link PixelExecutionUtils.AutomationCancelledException} so that + * existing callers importing this top-level class continue to compile unmodified. + * + *

The canonical definition is now the inner class + * {@link PixelExecutionUtils.AutomationCancelledException}. This file will be removed in a later + * cleanup pass once all callers (in particular the {@code nodes/} sub-package files) are updated + * to reference the inner class directly. + * + * @deprecated Use {@link PixelExecutionUtils.AutomationCancelledException} directly. + */ +@Deprecated +public class AutomationCancelledException extends PixelExecutionUtils.AutomationCancelledException { + + private static final long serialVersionUID = 1L; + + public AutomationCancelledException(String message) { + super(message); + } +} diff --git a/src/prerna/reactor/automation/AutomationConstants.java b/src/prerna/reactor/automation/AutomationConstants.java new file mode 100644 index 00000000000..850c1145551 --- /dev/null +++ b/src/prerna/reactor/automation/AutomationConstants.java @@ -0,0 +1,258 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +public final class AutomationConstants { + + private AutomationConstants() {} + + // -- File names ---------------------------------------------------------------- + + public static final String AUTOMATION_FILE_NAME = "automation.json"; + public static final String AUTOMATION_CONFIG_FILE_NAME = "automation-config.json"; + + public static final String SENSITIVE_MASK = "***"; + + // -- DB Table names ------------------------------------------------------------ + + public static final String TABLE_AUTOMATION_RUNS = "AUTOMATION_RUNS"; + public static final String TABLE_AUTOMATION_NODE_OUTPUTS = "AUTOMATION_NODE_OUTPUTS"; + public static final String TABLE_AUTOMATION_ACTIVE_RUN = "AUTOMATION_ACTIVE_RUN"; + + // -- AUTOMATION_RUNS columns --------------------------------------------------- + + public static final String RUN_ID = "RUN_ID"; + public static final String PROJECT_ID = "PROJECT_ID"; + public static final String AUTOMATION_ID = "AUTOMATION_ID"; + public static final String DEFINITION_VERSION = "DEFINITION_VERSION"; + public static final String DEFINITION_HASH = "DEFINITION_HASH"; + public static final String DEFINITION_SNAPSHOT = "DEFINITION_SNAPSHOT"; + public static final String STATUS = "STATUS"; + public static final String TRIGGER_TYPE = "TRIGGER_TYPE"; + public static final String STARTED_AT = "STARTED_AT"; + public static final String COMPLETED_AT = "COMPLETED_AT"; + public static final String FAILED_NODE_ID = "FAILED_NODE_ID"; + public static final String ERROR_MESSAGE = "ERROR_MESSAGE"; + public static final String LAST_HEARTBEAT = "LAST_HEARTBEAT"; + public static final String TOTAL_NODES = "TOTAL_NODES"; + public static final String COMPLETED_NODES = "COMPLETED_NODES"; + public static final String CREATED_BY = "CREATED_BY"; + public static final String CANCEL_REQUESTED = "CANCEL_REQUESTED"; + public static final String RESULT_SUMMARY_COL = "RESULT_SUMMARY"; + + // -- AUTOMATION_ACTIVE_RUN columns --------------------------------------------- + + public static final String CLAIMED_AT = "CLAIMED_AT"; + + // -- AUTOMATION_NODE_OUTPUTS columns ------------------------------------------ + + public static final String NODE_ID = "NODE_ID"; + public static final String NODE_LABEL = "NODE_LABEL"; + public static final String EXECUTION_ORDER = "EXECUTION_ORDER"; + public static final String DURATION_MS = "DURATION_MS"; + public static final String OUTPUT_VAR = "OUTPUT_VAR"; + public static final String OUTPUT_VALUE = "OUTPUT_VALUE"; + public static final String OUTPUT_PREVIEW = "OUTPUT_PREVIEW"; + + // -- Run statuses -------------------------------------------------------------- + + public static final String STATUS_RUNNING = "RUNNING"; + public static final String STATUS_SUCCESS = "SUCCESS"; + public static final String STATUS_FAILED = "FAILED"; + public static final String STATUS_INTERRUPTED = "INTERRUPTED"; + public static final String STATUS_CANCELLED = "CANCELLED"; + + // -- Node statuses ------------------------------------------------------------- + + public static final String NODE_STATUS_PENDING = "PENDING"; + public static final String NODE_STATUS_RUNNING = "RUNNING"; + public static final String NODE_STATUS_SUCCESS = "SUCCESS"; + public static final String NODE_STATUS_FAILED = "FAILED"; + public static final String NODE_STATUS_SKIPPED = "SKIPPED"; + + // -- Trigger types ------------------------------------------------------------- + + public static final String TRIGGER_MANUAL = "MANUAL"; + public static final String TRIGGER_PLAYGROUND = "PLAYGROUND"; + + // -- Node types (Phase 1) ------------------------------------------------------ + + public static final String NODE_TRIGGER = "trigger"; + public static final String NODE_DATABASE_ENGINE = "database-engine"; + public static final String NODE_STORAGE_ENGINE = "storage-engine"; + public static final String NODE_VECTOR_ENGINE = "vector-engine"; + public static final String NODE_MODEL_ENGINE = "model-engine"; + public static final String NODE_FUNCTION_ENGINE = "function-engine"; + public static final String NODE_WAIT = "wait"; + public static final String NODE_APP = "app"; + + // -- Node config keys (node.config map fields, shared across executors) -------- + + public static final String CONFIG_ENGINE_ID = "engineId"; + public static final String CONFIG_OPERATION = "operation"; + public static final String CONFIG_EXPRESSION = "expression"; + public static final String CONFIG_LIMIT = "limit"; + public static final String CONFIG_VALUES = "values"; + public static final String CONFIG_COMMAND = "command"; + public static final String CONFIG_CONTEXT = "context"; + public static final String CONFIG_PARAM_VALUES = "paramValues"; + public static final String CONFIG_PARAMS = "params"; + public static final String CONFIG_STORAGE_PATH = "storagePath"; + public static final String CONFIG_FILE_PATH = "filePath"; + public static final String CONFIG_FILE_NAMES = "fileNames"; + public static final String CONFIG_SECONDS = "seconds"; + public static final String CONFIG_PIXEL = "pixel"; + public static final String CONFIG_APP_ID = "appId"; + public static final String CONFIG_TIMEOUT_SECONDS = "timeoutSeconds"; + public static final String DEFAULT_STORAGE_PATH = "/"; + public static final String EMPTY_JSON_OBJECT = "{}"; + public static final String EMPTY_JSON_ARRAY = "[]"; + + // -- Node operation values ------------------------------------------------------- + + public static final String OP_READ = "read"; + public static final String OP_WRITE = "write"; + public static final String OP_LLM = "llm"; + public static final String OP_EMBEDDINGS = "embeddings"; + public static final String OP_VISION = "vision"; + public static final String OP_NER = "ner"; + public static final String OP_SEARCH = "search"; + public static final String OP_ADD_FILE = "add-file"; + public static final String OP_ADD_CSV = "add-csv"; + public static final String OP_LIST = "list"; + public static final String OP_DELETE = "delete"; + public static final String OP_DOWNLOAD = "download"; + public static final String OP_UPLOAD = "upload"; + public static final String OP_READ_BASE64 = "read-base64"; + + // -- Node execution defaults / bounds -------------------------------------------- + + public static final int DEFAULT_DB_QUERY_LIMIT = 50; + public static final int DEFAULT_VECTOR_SEARCH_LIMIT = 5; + public static final int DEFAULT_LIST_RUNS_LIMIT = 25; + public static final int WAIT_MIN_SECONDS = 0; + public static final int WAIT_MAX_SECONDS = 3600; + public static final int WAIT_DEFAULT_SECONDS = 1; + public static final int WAIT_CANCEL_CHECK_INTERVAL_SECONDS = 5; + + // -- automation.json document field names ---------------------------------------- + + public static final String DOC_VERSION = "version"; + public static final String DOC_GRAPH = "graph"; + public static final String DOC_NODES = "nodes"; + public static final String DOC_EDGES = "edges"; + public static final int DOC_CURRENT_VERSION = 1; + public static final String DOC_RESULT_MESSAGE_TEMPLATE = "resultMessageTemplate"; + public static final String DOC_DESCRIPTION = "description"; + + // -- Node/edge field names -------------------------------------------------------- + + public static final String NODE_FIELD_ID = "id"; + public static final String NODE_FIELD_TYPE = "type"; + public static final String NODE_FIELD_LABEL = "label"; + public static final String NODE_FIELD_CONFIG = "config"; + public static final String NODE_FIELD_OUTPUT_TRANSFORM = "outputTransform"; + public static final String NODE_FIELD_OUTPUT_VAR = "outputVar"; + public static final String EDGE_FIELD_SOURCE = "source"; + public static final String EDGE_FIELD_TARGET = "target"; + public static final String UNNAMED_NODE_LABEL = "unnamed"; + + // -- automation-config.json entry field names ------------------------------------ + + public static final String CONFIG_ENTRY_KEY = "key"; + public static final String CONFIG_ENTRY_VALUE = "value"; + public static final String CONFIG_ENTRY_SENSITIVE = "sensitive"; + + // -- Output transform field names / modes ---------------------------------------- + + public static final String TRANSFORM_MODE = "mode"; + public static final String TRANSFORM_COLUMN = "column"; + public static final String TRANSFORM_PATH = "path"; + public static final String TRANSFORM_MODE_RAW = "raw"; + public static final String TRANSFORM_MODE_ROWS_AS_OBJECTS = "rows-as-objects"; + public static final String TRANSFORM_MODE_FIRST_ROW = "first-row"; + public static final String TRANSFORM_MODE_COLUMN = "column"; + public static final String TRANSFORM_MODE_JSONPATH = "jsonpath"; + public static final String DATASET_HEADERS = "headers"; + public static final String DATASET_VALUES = "values"; + public static final String DATASET_DATA = "data"; + + // -- Scope variable names --------------------------------------------------------- + + public static final String SCOPE_DATE = "date"; + public static final String SCOPE_TRIGGERED_AT = "triggered_at"; + public static final String SCOPE_RUN_ID = "run_id"; + public static final String TEST_RUN_ID = "test"; + public static final String SYSTEM_USER_ID = "system"; + + // -- Result map keys --------------------------------------------------------------- + + public static final String RESULT_NODE_RESULTS = "nodeResults"; + public static final String RESULT_CANCEL_REQUESTED = "cancelRequested"; + public static final String RESULT_SIGNALLED_LOCALLY = "signalledLocally"; + public static final String RESULT_OUTPUT_VALUE = "outputValue"; + /** Human-readable, per-workflow summary surfaced to MCP/agent consumers (see {@link #DOC_RESULT_MESSAGE_TEMPLATE}). */ + public static final String RESULT_SUMMARY = "summary"; + /** Enriched summary including per-step output previews, sent as the MCP tool response so the LLM can describe what happened. */ + public static final String RESULT_LLM_CONTEXT = "llmContext"; + + // -- Pixel execution defaults ---------------------------------------------------- + + public static final String AUTOMATION_INPUTS_KEY = "inputs"; + public static final String AUTOMATION_TRIGGER_TYPE_KEY = "triggerType"; + public static final int DEFAULT_TIMEOUT_SECONDS = 300; + + // -- Data type constants (for table creation) ---------------------------------- + + public static final String VARCHAR_50 = "VARCHAR(50)"; + public static final String VARCHAR_255 = "VARCHAR(255)"; + public static final String VARCHAR_500 = "VARCHAR(500)"; + public static final String VARCHAR_2000 = "VARCHAR (2000)"; + public static final String INTEGER = "INTEGER"; + public static final String BIGINT = "BIGINT"; + public static final String NOT_NULL = "NOT NULL"; + + // -- DDL object names (indexes / primary keys) ---------------------------------- + + public static final String PK_AUTOMATION_RUNS = "PK_AUTOMATION_RUNS"; + public static final String PK_AUTO_NODE_OUT = "PK_AUTO_NODE_OUT"; + public static final String PK_AUTO_ACTIVE_RUN = "PK_AUTO_ACTIVE_RUN"; + public static final String IDX_AR_PROJECT = "IDX_AR_PROJECT"; + public static final String IDX_AR_STATUS = "IDX_AR_STATUS"; + public static final String IDX_AR_STARTED = "IDX_AR_STARTED"; + public static final String IDX_ANO_RUN = "IDX_ANO_RUN"; + + // -- Defaults ------------------------------------------------------------------ + + public static final String DEFAULT_AUTOMATION_ID = "default"; + public static final int HEARTBEAT_INTERVAL_SECONDS = 30; + public static final int STALE_HEARTBEAT_THRESHOLD_MINUTES = 5; + public static final int OUTPUT_PREVIEW_MAX_LENGTH = 2000; + /** Maximum characters from a node output preview included in the AI run-summary prompt. */ + public static final int SUMMARY_PROMPT_PREVIEW_MAX_LENGTH = 300; +} diff --git a/src/prerna/reactor/automation/AutomationDatabaseUtility.java b/src/prerna/reactor/automation/AutomationDatabaseUtility.java new file mode 100644 index 00000000000..054256bf9bb --- /dev/null +++ b/src/prerna/reactor/automation/AutomationDatabaseUtility.java @@ -0,0 +1,1226 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +import static prerna.reactor.automation.AutomationConstants.AUTOMATION_ID; +import static prerna.reactor.automation.AutomationConstants.BIGINT; +import static prerna.reactor.automation.AutomationConstants.CANCEL_REQUESTED; +import static prerna.reactor.automation.AutomationConstants.RESULT_SUMMARY_COL; +import static prerna.reactor.automation.AutomationConstants.CLAIMED_AT; +import static prerna.reactor.automation.AutomationConstants.COMPLETED_AT; +import static prerna.reactor.automation.AutomationConstants.COMPLETED_NODES; +import static prerna.reactor.automation.AutomationConstants.CREATED_BY; +import static prerna.reactor.automation.AutomationConstants.DEFINITION_HASH; +import static prerna.reactor.automation.AutomationConstants.DEFINITION_SNAPSHOT; +import static prerna.reactor.automation.AutomationConstants.DEFINITION_VERSION; +import static prerna.reactor.automation.AutomationConstants.DURATION_MS; +import static prerna.reactor.automation.AutomationConstants.ERROR_MESSAGE; +import static prerna.reactor.automation.AutomationConstants.EXECUTION_ORDER; +import static prerna.reactor.automation.AutomationConstants.FAILED_NODE_ID; +import static prerna.reactor.automation.AutomationConstants.IDX_ANO_RUN; +import static prerna.reactor.automation.AutomationConstants.IDX_AR_PROJECT; +import static prerna.reactor.automation.AutomationConstants.IDX_AR_STARTED; +import static prerna.reactor.automation.AutomationConstants.IDX_AR_STATUS; +import static prerna.reactor.automation.AutomationConstants.INTEGER; +import static prerna.reactor.automation.AutomationConstants.LAST_HEARTBEAT; +import static prerna.reactor.automation.AutomationConstants.NODE_FIELD_ID; +import static prerna.reactor.automation.AutomationConstants.NODE_FIELD_LABEL; +import static prerna.reactor.automation.AutomationConstants.NODE_ID; +import static prerna.reactor.automation.AutomationConstants.NODE_LABEL; +import static prerna.reactor.automation.AutomationConstants.NODE_STATUS_FAILED; +import static prerna.reactor.automation.AutomationConstants.NODE_STATUS_PENDING; +import static prerna.reactor.automation.AutomationConstants.NODE_STATUS_RUNNING; +import static prerna.reactor.automation.AutomationConstants.NODE_STATUS_SKIPPED; +import static prerna.reactor.automation.AutomationConstants.NODE_STATUS_SUCCESS; +import static prerna.reactor.automation.AutomationConstants.NOT_NULL; +import static prerna.reactor.automation.AutomationConstants.OUTPUT_PREVIEW; +import static prerna.reactor.automation.AutomationConstants.OUTPUT_VALUE; +import static prerna.reactor.automation.AutomationConstants.OUTPUT_VAR; +import static prerna.reactor.automation.AutomationConstants.PK_AUTOMATION_RUNS; +import static prerna.reactor.automation.AutomationConstants.PK_AUTO_ACTIVE_RUN; +import static prerna.reactor.automation.AutomationConstants.PK_AUTO_NODE_OUT; +import static prerna.reactor.automation.AutomationConstants.PROJECT_ID; +import static prerna.reactor.automation.AutomationConstants.RUN_ID; +import static prerna.reactor.automation.AutomationConstants.STALE_HEARTBEAT_THRESHOLD_MINUTES; +import static prerna.reactor.automation.AutomationConstants.STARTED_AT; +import static prerna.reactor.automation.AutomationConstants.STATUS; +import static prerna.reactor.automation.AutomationConstants.STATUS_INTERRUPTED; +import static prerna.reactor.automation.AutomationConstants.STATUS_RUNNING; +import static prerna.reactor.automation.AutomationConstants.TABLE_AUTOMATION_ACTIVE_RUN; +import static prerna.reactor.automation.AutomationConstants.TABLE_AUTOMATION_NODE_OUTPUTS; +import static prerna.reactor.automation.AutomationConstants.TABLE_AUTOMATION_RUNS; +import static prerna.reactor.automation.AutomationConstants.TOTAL_NODES; +import static prerna.reactor.automation.AutomationConstants.TRIGGER_TYPE; +import static prerna.reactor.automation.AutomationConstants.VARCHAR_2000; +import static prerna.reactor.automation.AutomationConstants.VARCHAR_255; +import static prerna.reactor.automation.AutomationConstants.VARCHAR_50; +import static prerna.reactor.automation.AutomationConstants.VARCHAR_500; + +import java.io.UnsupportedEncodingException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.engine.api.IRDBMSEngine; +import prerna.query.querystruct.SelectQueryStruct; +import prerna.query.querystruct.filters.SimpleQueryFilter; +import prerna.query.querystruct.selectors.QueryColumnOrderBySelector; +import prerna.query.querystruct.selectors.QueryColumnSelector; +import prerna.sablecc2.om.PixelDataType; +import prerna.util.ConnectionUtils; +import prerna.util.QueryExecutionUtility; +import prerna.util.SystemEngineRegistry; +import prerna.util.Utility; +import prerna.util.sql.AbstractSqlQueryUtil; + +/** + * Database utility for the Automation Engine subsystem. + * Manages AUTOMATION_RUNS, AUTOMATION_NODE_OUTPUTS, and AUTOMATION_ACTIVE_RUN tables + * in the scheduler database. + * + * Follows the same patterns as {@link prerna.reactor.scheduler.SchedulerDatabaseUtility}. + * Called at platform startup to create tables; provides CRUD for automation execution state. + */ +public final class AutomationDatabaseUtility { + + private static final Logger classLogger = LogManager.getLogger(AutomationDatabaseUtility.class); + + // Table name shortcuts for SelectQueryStruct (TABLE__COLUMN format) + private static final String TABLE_RUNS = TABLE_AUTOMATION_RUNS; + private static final String TABLE_NODE_OUTPUTS = TABLE_AUTOMATION_NODE_OUTPUTS; + private static final String TABLE_ACTIVE_RUN = TABLE_AUTOMATION_ACTIVE_RUN; + + private AutomationDatabaseUtility() { + // static utility - no instantiation + } + + // -- SQL Statements (INSERT/UPDATE/DELETE - PreparedStatement per SEMOSS conventions) -- + + // AUTOMATION_RUNS + private static final String INSERT_RUN = """ + INSERT INTO AUTOMATION_RUNS \ + (RUN_ID, PROJECT_ID, AUTOMATION_ID, DEFINITION_VERSION, DEFINITION_HASH, DEFINITION_SNAPSHOT, \ + STATUS, TRIGGER_TYPE, \ + STARTED_AT, LAST_HEARTBEAT, TOTAL_NODES, COMPLETED_NODES, CREATED_BY) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?)"""; + + private static final String UPDATE_RUN_STATUS = """ + UPDATE AUTOMATION_RUNS SET STATUS = ?, COMPLETED_AT = ?, \ + FAILED_NODE_ID = ?, ERROR_MESSAGE = ? WHERE RUN_ID = ?"""; + + private static final String UPDATE_RUN_SUMMARY = + "UPDATE AUTOMATION_RUNS SET RESULT_SUMMARY = ? WHERE RUN_ID = ?"; + + private static final String UPDATE_HEARTBEAT = + "UPDATE AUTOMATION_RUNS SET LAST_HEARTBEAT = ?, COMPLETED_NODES = ? WHERE RUN_ID = ?"; + + private static final String TOUCH_HEARTBEAT = + "UPDATE AUTOMATION_RUNS SET LAST_HEARTBEAT = ? WHERE RUN_ID = ?"; + + private static final String SET_CANCEL_REQUESTED = + "UPDATE AUTOMATION_RUNS SET CANCEL_REQUESTED = ? WHERE RUN_ID = ?"; + + // AUTOMATION_ACTIVE_RUN - single row per project, PK on PROJECT_ID enforces exclusivity + private static final String CLAIM_ACTIVE_RUN = + "INSERT INTO AUTOMATION_ACTIVE_RUN (PROJECT_ID, RUN_ID, CLAIMED_AT) VALUES (?, ?, ?)"; + + private static final String RELEASE_ACTIVE_RUN = + "DELETE FROM AUTOMATION_ACTIVE_RUN WHERE PROJECT_ID = ? AND RUN_ID = ?"; + + private static final String MARK_STALE_INTERRUPTED = """ + UPDATE AUTOMATION_RUNS SET STATUS = ?, COMPLETED_AT = ?, \ + ERROR_MESSAGE = ? WHERE RUN_ID = ?"""; + + // AUTOMATION_NODE_OUTPUTS + private static final String INSERT_NODE_OUTPUT = """ + INSERT INTO AUTOMATION_NODE_OUTPUTS \ + (RUN_ID, NODE_ID, NODE_LABEL, EXECUTION_ORDER, STATUS) \ + VALUES (?, ?, ?, ?, ?)"""; + + private static final String UPDATE_NODE_OUTPUT_SUCCESS = """ + UPDATE AUTOMATION_NODE_OUTPUTS SET STATUS = ?, STARTED_AT = ?, COMPLETED_AT = ?, \ + DURATION_MS = ?, OUTPUT_VAR = ?, OUTPUT_VALUE = ?, OUTPUT_PREVIEW = ? \ + WHERE RUN_ID = ? AND NODE_ID = ?"""; + + private static final String UPDATE_NODE_OUTPUT_FAILED = """ + UPDATE AUTOMATION_NODE_OUTPUTS SET STATUS = ?, STARTED_AT = ?, COMPLETED_AT = ?, \ + DURATION_MS = ?, ERROR_MESSAGE = ? WHERE RUN_ID = ? AND NODE_ID = ?"""; + + private static final String UPDATE_NODE_STATUS = + "UPDATE AUTOMATION_NODE_OUTPUTS SET STATUS = ?, STARTED_AT = ? WHERE RUN_ID = ? AND NODE_ID = ?"; + + private static final String SKIP_PENDING_NODE_OUTPUTS = + "UPDATE AUTOMATION_NODE_OUTPUTS SET STATUS = ?, ERROR_MESSAGE = ? WHERE RUN_ID = ? AND STATUS = ?"; + + // -- Initialization ------------------------------------------------------------ + + /** + * Creates automation tables in the scheduler DB if they don't exist, and + * registers them in the OWL. Called at platform startup after the scheduler + * DB is loaded. Safe to call on every startup (uses IF NOT EXISTS / metadata + * checks). + */ + public static void initialize() { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) { + classLogger.warn("Scheduler DB not available - automation tables will not be created"); + return; + } + + // Register the automation OWL schema in the scheduler DB if any tables or + // columns are missing. This keeps the OWL declaration entirely within this + // package rather than depending on SchedulerOwlCreator. + AutomationOwlCreator owlCreator = new AutomationOwlCreator(); + if (owlCreator.needsRemake(schedulerDb)) { + try { + owlCreator.remakeOwl(schedulerDb); + } catch (Exception e) { + classLogger.error("Failed to update automation OWL schema in scheduler DB", e); + } + } + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + AbstractSqlQueryUtil queryUtil = schedulerDb.getQueryUtil(); + String database = schedulerDb.getDatabase(); + String schema = schedulerDb.getSchema(); + + boolean allowIfExists = queryUtil.allowsIfExistsTableSyntax(); + String dateTimeType = queryUtil.getDateWithTimeDataType(); + String clobType = queryUtil.getClobDataTypeName(); + + createAutomationRunsTable(conn, queryUtil, database, schema, allowIfExists, dateTimeType, clobType); + createAutomationNodeOutputsTable(conn, queryUtil, database, schema, allowIfExists, dateTimeType, clobType); + createAutomationActiveRunTable(conn, queryUtil, database, schema, allowIfExists, dateTimeType); + + if (!conn.getAutoCommit()) { + conn.commit(); + } + + classLogger.info("Automation engine tables initialized successfully"); + } catch (Exception e) { + classLogger.error("Failed to initialize automation engine tables", e); + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * On startup, marks any runs stuck in RUNNING as INTERRUPTED. + * This handles the case where the server crashed mid-execution. + */ + public static void markStaleRunsInterrupted() { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return; + + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + RUN_ID, RUN_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + PROJECT_ID, PROJECT_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + LAST_HEARTBEAT, LAST_HEARTBEAT)); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_RUNS + "__" + STATUS, "==", STATUS_RUNNING, PixelDataType.CONST_STRING)); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + if (results == null || results.isEmpty()) { + return; + } + + Timestamp threshold = toTimestamp(Instant.now().minusSeconds( + STALE_HEARTBEAT_THRESHOLD_MINUTES * 60L)); + Timestamp now = toTimestamp(Instant.now()); + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + for (Map row : results) { + String runId = (String) row.get(RUN_ID); + String projectId = (String) row.get(PROJECT_ID); + + // Only interrupt runs whose heartbeat is actually stale. A run with a fresh + // heartbeat is still alive (e.g. executing on another node in a cluster), so + // interrupting it would clobber active work. A missing/unparseable heartbeat + // is treated as stale (a crashed run that never checkpointed). + Timestamp lastHeartbeat = toTimestampSafe(row.get(LAST_HEARTBEAT)); + if (lastHeartbeat != null && lastHeartbeat.after(threshold)) { + classLogger.debug("Skipping automation run {} - heartbeat {} is newer than stale threshold {}", + runId, lastHeartbeat, threshold); + continue; + } + + try (PreparedStatement ps = conn.prepareStatement(MARK_STALE_INTERRUPTED)) { + int index = 1; + ps.setString(index++, STATUS_INTERRUPTED); + ps.setTimestamp(index++, now); + ps.setString(index++, "Server restarted during execution"); + ps.setString(index++, runId); + int updated = ps.executeUpdate(); + if (updated > 0) { + classLogger.info("Marked stale automation run {} as INTERRUPTED", runId); + } + } + // Release the active-run slot so the project can be re-triggered - otherwise a + // crashed run would permanently block that project from ever running again. + if (projectId != null) { + releaseActiveRun(projectId, runId); + } + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + } catch (Exception e) { + classLogger.error("Failed to mark stale automation runs", e); + } finally { + closeConnection(schedulerDb, conn); + } + } + + // -- AUTOMATION_RUNS CRUD -------------------------------------------------------- + + /** + * Checks if an automation already has an active (RUNNING) run for the given project. + * + * @return the active run ID, or null if no run is active + */ + public static String getActiveRun(String projectId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return null; + + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + RUN_ID, RUN_ID)); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_RUNS + "__" + PROJECT_ID, "==", projectId, PixelDataType.CONST_STRING)); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_RUNS + "__" + STATUS, "==", STATUS_RUNNING, PixelDataType.CONST_STRING)); + qs.setLimit(1); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + if (results != null && !results.isEmpty()) { + Object runId = results.get(0).get(RUN_ID); + return runId != null ? runId.toString() : null; + } + return null; + } + + /** + * Atomically claims the "active run" slot for a project. Backed by a single-row-per-project + * marker table ({@code AUTOMATION_ACTIVE_RUN}, PK on {@code PROJECT_ID}) in the shared scheduler + * DB, so this is correct across every pod in a cluster - not just within one JVM. Unlike + * {@link #getActiveRun(String)} (a plain SELECT), this is a single atomic INSERT: a PK + * violation means another run is already active for that project, closing the check-then-insert + * race where two concurrent triggers for the same project could otherwise both start a run and + * double up any node with side effects (e.g. a database-update node running twice). + * + *

If the scheduler DB is unavailable, fails open (returns true) to match the existing + * degraded-mode behavior of the rest of this class (e.g. {@link #insertRun}, which silently + * no-ops when the scheduler DB can't be reached) rather than introduce a new failure mode. + * + * @return true if the slot was claimed (caller may proceed), false if another run already + * holds it for this project + */ + public static boolean claimActiveRun(String projectId, String runId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) { + classLogger.warn("Scheduler DB not available - cannot enforce single-active-run guard for project {}", projectId); + return true; + } + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(CLAIM_ACTIVE_RUN)) { + int index = 1; + ps.setString(index++, projectId); + ps.setString(index++, runId); + ps.setTimestamp(index++, toTimestamp(Instant.now())); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + // Constraint violation (another run already holds this project's slot) is the + // expected/common case here, not an error - log at debug, not error. + classLogger.debug("Could not claim active-run slot for project {} (likely already active): {}", + projectId, e.getMessage()); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Releases the "active run" slot for a project, allowing a new run to be claimed. + * Must be called on every terminal run status (SUCCESS/FAILED/CANCELLED/INTERRUPTED), + * including the stale-run sweep in {@link #markStaleRunsInterrupted()}. + */ + public static boolean releaseActiveRun(String projectId, String runId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(RELEASE_ACTIVE_RUN)) { + ps.setString(1, projectId); + ps.setString(2, runId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to release active-run slot for project {}, run {}", + projectId, runId, e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Returns the active run ID for a project directly from the {@code AUTOMATION_ACTIVE_RUN} lock + * table. Unlike {@link #getActiveRun(String)}, this is populated at {@link #claimActiveRun} time + * — before {@code AUTOMATION_RUNS} is written — so callers polling for a newly started run will + * see it sooner. + * + * @return the run ID, or {@code null} if no run is currently active for the project + */ + public static String getClaimedActiveRun(String projectId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return null; + + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector(TABLE_ACTIVE_RUN + "__" + RUN_ID, RUN_ID)); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_ACTIVE_RUN + "__" + PROJECT_ID, "==", projectId, PixelDataType.CONST_STRING)); + qs.setLimit(1); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + if (results != null && !results.isEmpty()) { + Object runId = results.get(0).get(RUN_ID); + return runId != null ? runId.toString() : null; + } + return null; + } + + /** + * Sets the cluster-safe cancellation flag on a run. Called by {@code CancelAutomationRunReactor} + * regardless of which pod receives the cancel request - unlike the in-memory + * {@code TriggerAutomationReactor.CANCELLATION_FLAGS} map (a same-pod-only fast path), this is + * visible to whichever pod is actually executing the run via {@link #isCancelRequested(String)}. + */ + public static boolean setCancelRequested(String runId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(SET_CANCEL_REQUESTED)) { + ps.setBoolean(1, true); + ps.setString(2, runId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to set cancel-requested flag for run '{}'", runId, e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Checks the cluster-safe cancellation flag for a run. Polled by the executing pod's + * between-node cancellation check in addition to the local in-memory flag, so a cancel + * request landing on a different pod than the one executing the run is still honored. + */ + public static boolean isCancelRequested(String runId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + CANCEL_REQUESTED, + CANCEL_REQUESTED)); + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_RUNS + "__" + RUN_ID, "==", runId, PixelDataType.CONST_STRING)); + qs.setLimit(1); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + if (results == null || results.isEmpty()) { + return false; + } + Object flag = results.get(0).get(CANCEL_REQUESTED); + if (flag instanceof Boolean) { + return (Boolean) flag; + } + return flag != null && Boolean.parseBoolean(flag.toString()); + } + + /** + * Inserts a new automation run record. + */ + public static boolean insertRun(String runId, String projectId, String automationId, + int definitionVersion, String definitionHash, String definitionSnapshot, + String triggerType, int totalNodes, String createdBy) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + AbstractSqlQueryUtil queryUtil = schedulerDb.getQueryUtil(); + Timestamp now = toTimestamp(Instant.now()); + + try (PreparedStatement ps = conn.prepareStatement(INSERT_RUN)) { + int index = 1; + ps.setString(index++, runId); + ps.setString(index++, projectId); + ps.setString(index++, automationId); + ps.setInt(index++, definitionVersion); + ps.setString(index++, definitionHash); + queryUtil.handleInsertionOfClob(conn, ps, definitionSnapshot, index++, AutomationExecutionUtils.GSON); + ps.setString(index++, STATUS_RUNNING); + ps.setString(index++, triggerType); + ps.setTimestamp(index++, now); + ps.setTimestamp(index++, now); + ps.setInt(index++, totalNodes); + ps.setString(index++, createdBy); + ps.executeUpdate(); + } + + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException | UnsupportedEncodingException e) { + classLogger.error("Failed to insert automation run '{}'", runId, e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Updates the status of an automation run (on completion or failure). + */ + public static boolean updateRunStatus(String runId, String status, + String failedNodeId, String errorMessage) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(UPDATE_RUN_STATUS)) { + int index = 1; + ps.setString(index++, status); + ps.setTimestamp(index++, toTimestamp(Instant.now())); + setNullableString(ps, index++, failedNodeId); + setNullableString(ps, index++, errorMessage); + ps.setString(index++, runId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to update run status for '{}'", runId, e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Persists the human-readable outcome summary for a completed run. + * Called after the run finishes, separately from {@link #updateRunStatus} because the + * summary is built by the caller ({@code TriggerAutomationReactor}) after the engine returns. + */ + public static boolean updateRunSummary(String runId, String resultSummary) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(UPDATE_RUN_SUMMARY)) { + setNullableString(ps, 1, resultSummary); + ps.setString(2, runId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to update run summary for '{}'", runId, e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Updates the heartbeat timestamp and completed node count for a running automation. + */ + public static boolean updateHeartbeat(String runId, int completedNodes) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(UPDATE_HEARTBEAT)) { + int index = 1; + ps.setTimestamp(index++, toTimestamp(Instant.now())); + ps.setInt(index++, completedNodes); + ps.setString(index++, runId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to update heartbeat for run '{}'", runId, e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Updates only the heartbeat timestamp for a running automation. + * Used when the node count hasn't changed but liveness needs to be signaled. + */ + public static boolean touchHeartbeat(String runId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(TOUCH_HEARTBEAT)) { + ps.setTimestamp(1, toTimestamp(Instant.now())); + ps.setString(2, runId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to touch heartbeat for run '{}'", runId, e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Lists automation runs for a project, newest first. + * + * @param projectId the project to query + * @param limit max number of runs to return + * @return list of run summary maps + */ + public static List> getRunsForProject(String projectId, int limit) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return new ArrayList<>(); + + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + RUN_ID, RUN_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + PROJECT_ID, PROJECT_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + AUTOMATION_ID, AUTOMATION_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + DEFINITION_VERSION, DEFINITION_VERSION)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + DEFINITION_HASH, DEFINITION_HASH)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + DEFINITION_SNAPSHOT, DEFINITION_SNAPSHOT)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + STATUS, STATUS)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + TRIGGER_TYPE, TRIGGER_TYPE)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + STARTED_AT, STARTED_AT)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + COMPLETED_AT, COMPLETED_AT)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + FAILED_NODE_ID, FAILED_NODE_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + TOTAL_NODES, TOTAL_NODES)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + COMPLETED_NODES, COMPLETED_NODES)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + CREATED_BY, CREATED_BY)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + RESULT_SUMMARY_COL, RESULT_SUMMARY_COL)); + + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_RUNS + "__" + PROJECT_ID, "==", projectId, PixelDataType.CONST_STRING)); + qs.addOrderBy(TABLE_RUNS + "__" + STARTED_AT, + QueryColumnOrderBySelector.ORDER_BY_DIRECTION.DESC.toString()); + qs.setLimit(limit); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + return results != null ? results : new ArrayList<>(); + } + + /** + * Gets a single run detail by run ID (includes error message). + */ + public static Map getRunDetail(String runId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return null; + + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + RUN_ID, RUN_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + PROJECT_ID, PROJECT_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + AUTOMATION_ID, AUTOMATION_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + DEFINITION_VERSION, DEFINITION_VERSION)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + DEFINITION_HASH, DEFINITION_HASH)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + DEFINITION_SNAPSHOT, DEFINITION_SNAPSHOT)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + STATUS, STATUS)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + TRIGGER_TYPE, TRIGGER_TYPE)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + STARTED_AT, STARTED_AT)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + COMPLETED_AT, COMPLETED_AT)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + FAILED_NODE_ID, FAILED_NODE_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + ERROR_MESSAGE, ERROR_MESSAGE)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + TOTAL_NODES, TOTAL_NODES)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + COMPLETED_NODES, COMPLETED_NODES)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + CREATED_BY, CREATED_BY)); + qs.addSelector(new QueryColumnSelector(TABLE_RUNS + "__" + RESULT_SUMMARY_COL, RESULT_SUMMARY_COL)); + + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_RUNS + "__" + RUN_ID, "==", runId, PixelDataType.CONST_STRING)); + qs.setLimit(1); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + if (results != null && !results.isEmpty()) { + return results.get(0); + } + return null; + } + + // -- AUTOMATION_NODE_OUTPUTS CRUD ------------------------------------------------ + + /** + * Batch-inserts all node outputs for a run (all PENDING). + */ + public static boolean insertAllNodeOutputs(String runId, List> orderedNodes) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(INSERT_NODE_OUTPUT)) { + for (int i = 0; i < orderedNodes.size(); i++) { + Map node = orderedNodes.get(i); + int index = 1; + ps.setString(index++, runId); + ps.setString(index++, (String) node.get(NODE_FIELD_ID)); + ps.setString(index++, (String) node.get(NODE_FIELD_LABEL)); + ps.setInt(index++, i); + ps.setString(index++, NODE_STATUS_PENDING); + ps.addBatch(); + } + ps.executeBatch(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to batch-insert node outputs for run '{}'", runId, e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Marks a node as RUNNING (before pixel execution starts). + */ + public static boolean markNodeRunning(String runId, String nodeId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(UPDATE_NODE_STATUS)) { + int index = 1; + ps.setString(index++, NODE_STATUS_RUNNING); + ps.setTimestamp(index++, toTimestamp(Instant.now())); + ps.setString(index++, runId); + ps.setString(index++, nodeId); + ps.executeUpdate(); + } + + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to mark node running for run '{}', node '{}'", + runId, nodeId, e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Marks all nodes that did not start because the run reached a terminal state as skipped. + */ + public static boolean skipPendingNodes(String runId, String reason) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(SKIP_PENDING_NODE_OUTPUTS)) { + ps.setString(1, NODE_STATUS_SKIPPED); + setNullableString(ps, 2, reason); + ps.setString(3, runId); + ps.setString(4, NODE_STATUS_PENDING); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to skip pending nodes for run '{}'", runId, e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Updates a node output after successful execution. + */ + public static boolean updateNodeSuccess(String runId, String nodeId, Timestamp startedAt, + long durationMs, String outputVar, String outputValue, String outputPreview) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + AbstractSqlQueryUtil queryUtil = schedulerDb.getQueryUtil(); + + try (PreparedStatement ps = conn.prepareStatement(UPDATE_NODE_OUTPUT_SUCCESS)) { + int index = 1; + ps.setString(index++, NODE_STATUS_SUCCESS); + ps.setTimestamp(index++, startedAt); + ps.setTimestamp(index++, toTimestamp(Instant.now())); + ps.setLong(index++, durationMs); + ps.setString(index++, outputVar); + // Handle CLOB for potentially large output values + queryUtil.handleInsertionOfClob(conn, ps, outputValue, index++, AutomationExecutionUtils.GSON); + ps.setString(index++, outputPreview); + ps.setString(index++, runId); + ps.setString(index++, nodeId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (Exception e) { + classLogger.error("Failed to update node success for run '{}', node '{}'", + runId, nodeId, e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Updates a node output after failed execution. + */ + public static boolean updateNodeFailed(String runId, String nodeId, Timestamp startedAt, + long durationMs, String errorMessage) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return false; + + Connection conn = null; + try { + conn = schedulerDb.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(UPDATE_NODE_OUTPUT_FAILED)) { + int index = 1; + ps.setString(index++, NODE_STATUS_FAILED); + ps.setTimestamp(index++, startedAt); + ps.setTimestamp(index++, toTimestamp(Instant.now())); + ps.setLong(index++, durationMs); + setNullableString(ps, index++, errorMessage); + ps.setString(index++, runId); + ps.setString(index++, nodeId); + ps.executeUpdate(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + return true; + } catch (SQLException e) { + classLogger.error("Failed to update node failed for run '{}', node '{}'", + runId, nodeId, e); + return false; + } finally { + closeConnection(schedulerDb, conn); + } + } + + /** + * Gets all node outputs for a run, ordered by execution order. + */ + public static List> getNodeOutputsForRun(String runId) { + IRDBMSEngine schedulerDb = getSchedulerDb(); + if (schedulerDb == null) return new ArrayList<>(); + + SelectQueryStruct qs = new SelectQueryStruct(); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + NODE_ID, NODE_ID)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + NODE_LABEL, NODE_LABEL)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + EXECUTION_ORDER, EXECUTION_ORDER)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + STATUS, STATUS)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + STARTED_AT, STARTED_AT)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + COMPLETED_AT, COMPLETED_AT)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + DURATION_MS, DURATION_MS)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + OUTPUT_VAR, OUTPUT_VAR)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + OUTPUT_VALUE, OUTPUT_VALUE)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + OUTPUT_PREVIEW, OUTPUT_PREVIEW)); + qs.addSelector(new QueryColumnSelector(TABLE_NODE_OUTPUTS + "__" + ERROR_MESSAGE, ERROR_MESSAGE)); + + qs.addExplicitFilter(SimpleQueryFilter.makeColToValFilter( + TABLE_NODE_OUTPUTS + "__" + RUN_ID, "==", runId, PixelDataType.CONST_STRING)); + qs.addOrderBy(TABLE_NODE_OUTPUTS + "__" + EXECUTION_ORDER, + QueryColumnOrderBySelector.ORDER_BY_DIRECTION.ASC.toString()); + + List> results = QueryExecutionUtility.flushRsToMap(schedulerDb, qs); + return results != null ? results : new ArrayList<>(); + } + + // -- Result Assembly ----------------------------------------------------------- + + /** + * Builds a list of per-node result maps from the raw DB output rows returned by + * {@link #getNodeOutputsForRun(String)}. The shape matches what + * {@link prerna.reactor.automation.GetAutomationRunReactor} and + * {@link prerna.reactor.automation.TriggerAutomationReactor} return to callers. + * + *

Each entry contains: nodeId, nodeLabel, status, durationMs, outputPreview + * (falls back from outputValue when blank), outputValue, and errorMessage. + * + * @param nodeOutputs ordered rows from {@link #getNodeOutputsForRun(String)} + * @return mutable list of node result maps (empty when {@code nodeOutputs} is null) + */ + public static List> buildNodeResults(List> nodeOutputs) { + List> nodeResults = new ArrayList<>(); + if (nodeOutputs == null) { + return nodeResults; + } + for (Map output : nodeOutputs) { + Map nodeResult = new java.util.HashMap<>(); + nodeResult.put(AutomationConstants.NODE_ID, output.get(AutomationConstants.NODE_ID)); + nodeResult.put(AutomationConstants.NODE_LABEL, output.get(AutomationConstants.NODE_LABEL)); + nodeResult.put(AutomationConstants.STATUS, output.get(AutomationConstants.STATUS)); + nodeResult.put(AutomationConstants.DURATION_MS, output.get(AutomationConstants.DURATION_MS)); + String outputForDisplay = (String) output.get(AutomationConstants.OUTPUT_VALUE); + if (outputForDisplay == null || outputForDisplay.isBlank()) { + outputForDisplay = (String) output.get(AutomationConstants.OUTPUT_PREVIEW); + } + nodeResult.put(AutomationConstants.OUTPUT_PREVIEW, outputForDisplay); + nodeResult.put(AutomationConstants.OUTPUT_VALUE, output.get(AutomationConstants.OUTPUT_VALUE)); + nodeResult.put(AutomationConstants.ERROR_MESSAGE, output.get(AutomationConstants.ERROR_MESSAGE)); + nodeResults.add(nodeResult); + } + return nodeResults; + } + + // -- Table Creation ------------------------------------------------------------ + + private static void createAutomationRunsTable(Connection conn, AbstractSqlQueryUtil queryUtil, + String database, String schema, boolean allowIfExists, String dateTimeType, String clobType) throws SQLException { + + String tableName = TABLE_AUTOMATION_RUNS; + + boolean tableExists = !allowIfExists && queryUtil.tableExists(conn, tableName, database, schema); + if (!tableExists) { + String[] colNames = { RUN_ID, PROJECT_ID, AUTOMATION_ID, DEFINITION_VERSION, DEFINITION_HASH, + DEFINITION_SNAPSHOT, STATUS, TRIGGER_TYPE, STARTED_AT, COMPLETED_AT, FAILED_NODE_ID, + ERROR_MESSAGE, LAST_HEARTBEAT, TOTAL_NODES, COMPLETED_NODES, CREATED_BY, + CANCEL_REQUESTED, RESULT_SUMMARY_COL }; + String[] types = { VARCHAR_255, VARCHAR_255, VARCHAR_255, INTEGER, VARCHAR_255, + clobType, VARCHAR_50, VARCHAR_50, dateTimeType, dateTimeType, VARCHAR_255, + clobType, dateTimeType, INTEGER, INTEGER, VARCHAR_255, + queryUtil.getBooleanDataTypeName(), VARCHAR_2000 }; + String[] constraints = { NOT_NULL, NOT_NULL, null, null, null, + null, NOT_NULL, NOT_NULL, NOT_NULL, null, null, + null, null, null, null, null, + null, null }; + + String sql; + if (allowIfExists) { + sql = queryUtil.createTableIfNotExistsWithCustomConstraints(tableName, colNames, types, constraints); + } else { + sql = queryUtil.createTableWithCustomConstraints(tableName, colNames, types, constraints); + } + classLogger.info("Creating table {}: {}", tableName, sql); + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } + } + + // Additive migration for existing installations. + addColumnIfNotExists(conn, queryUtil, tableName, CANCEL_REQUESTED, queryUtil.getBooleanDataTypeName()); + addColumnIfNotExists(conn, queryUtil, tableName, RESULT_SUMMARY_COL, VARCHAR_2000); + addColumnIfNotExists(conn, queryUtil, tableName, DEFINITION_VERSION, INTEGER); + addColumnIfNotExists(conn, queryUtil, tableName, DEFINITION_HASH, VARCHAR_255); + addColumnIfNotExists(conn, queryUtil, tableName, DEFINITION_SNAPSHOT, clobType); + + // Primary key + addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, PK_AUTOMATION_RUNS, + new String[]{ RUN_ID }); + + // Indexes + createIndexIfNotExists(conn, queryUtil, allowIfExists, IDX_AR_PROJECT, tableName, + new String[]{ PROJECT_ID }); + createIndexIfNotExists(conn, queryUtil, allowIfExists, IDX_AR_STATUS, tableName, + new String[]{ PROJECT_ID, STATUS }); + createIndexIfNotExists(conn, queryUtil, allowIfExists, IDX_AR_STARTED, tableName, + new String[]{ PROJECT_ID, STARTED_AT }); + } + + private static void createAutomationNodeOutputsTable(Connection conn, AbstractSqlQueryUtil queryUtil, + String database, String schema, boolean allowIfExists, String dateTimeType, String clobType) throws SQLException { + + String tableName = TABLE_AUTOMATION_NODE_OUTPUTS; + + if (!allowIfExists && queryUtil.tableExists(conn, tableName, database, schema)) { + return; + } + + String[] colNames = { RUN_ID, NODE_ID, NODE_LABEL, EXECUTION_ORDER, STATUS, + STARTED_AT, COMPLETED_AT, DURATION_MS, OUTPUT_VAR, + OUTPUT_VALUE, OUTPUT_PREVIEW, ERROR_MESSAGE }; + String[] types = { VARCHAR_255, VARCHAR_255, VARCHAR_500, INTEGER, VARCHAR_50, + dateTimeType, dateTimeType, BIGINT, VARCHAR_255, + clobType, VARCHAR_2000, clobType }; + String[] constraints = { NOT_NULL, NOT_NULL, null, NOT_NULL, NOT_NULL, + null, null, null, null, + null, null, null }; + + String sql; + if (allowIfExists) { + sql = queryUtil.createTableIfNotExistsWithCustomConstraints(tableName, colNames, types, constraints); + } else { + sql = queryUtil.createTableWithCustomConstraints(tableName, colNames, types, constraints); + } + classLogger.info("Creating table {}: {}", tableName, sql); + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } + + // Composite primary key + addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, PK_AUTO_NODE_OUT, + new String[]{ RUN_ID, NODE_ID }); + + // Indexes + createIndexIfNotExists(conn, queryUtil, allowIfExists, IDX_ANO_RUN, tableName, + new String[]{ RUN_ID }); + } + + /** + * Creates the AUTOMATION_ACTIVE_RUN marker table - a single row per project, keyed on + * PROJECT_ID, used to atomically enforce "at most one active run per project" cluster-wide. + * See {@link #claimActiveRun(String, String)} / {@link #releaseActiveRun(String, String)}. + */ + private static void createAutomationActiveRunTable(Connection conn, AbstractSqlQueryUtil queryUtil, + String database, String schema, boolean allowIfExists, String dateTimeType) throws SQLException { + + String tableName = TABLE_AUTOMATION_ACTIVE_RUN; + + if (!allowIfExists && queryUtil.tableExists(conn, tableName, database, schema)) { + return; + } + + String[] colNames = { PROJECT_ID, RUN_ID, CLAIMED_AT }; + String[] types = { VARCHAR_255, VARCHAR_255, dateTimeType }; + String[] constraints = { NOT_NULL, NOT_NULL, NOT_NULL }; + + String sql; + if (allowIfExists) { + sql = queryUtil.createTableIfNotExistsWithCustomConstraints(tableName, colNames, types, constraints); + } else { + sql = queryUtil.createTableWithCustomConstraints(tableName, colNames, types, constraints); + } + classLogger.info("Creating table {}: {}", tableName, sql); + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } + + // Primary key on PROJECT_ID alone (not RUN_ID) is what makes claimActiveRun atomic: + // a second INSERT for the same project - from any pod - violates this constraint. + addPrimaryKeyIfNotExists(conn, queryUtil, tableName, database, schema, PK_AUTO_ACTIVE_RUN, + new String[]{ PROJECT_ID }); + } + + // -- Helpers ------------------------------------------------------------------- + + private static IRDBMSEngine getSchedulerDb() { + try { + return SystemEngineRegistry.getSchedulerDb(); + } catch (Exception e) { + classLogger.warn("Could not obtain scheduler DB: {}", e.getMessage()); + return null; + } + } + + private static void closeConnection(IRDBMSEngine engine, Connection conn) { + ConnectionUtils.closeAllConnectionsIfPooling(engine, conn); + } + + /** + * Binds a nullable VARCHAR column value, using {@code setNull(Types.VARCHAR)} instead of + * {@code setString(index, null)} when the value is absent - some JDBC drivers require an + * explicit SQL type for a null bind rather than inferring it from a null String argument. + */ + private static void setNullableString(PreparedStatement ps, int index, String value) throws SQLException { + if (value != null) { + ps.setString(index, value); + } else { + ps.setNull(index, Types.VARCHAR); + } + } + + private static Timestamp toTimestamp(Instant instant) { + return Utility.getSqlTimestampUTC( + LocalDateTime.ofInstant(instant, ZoneOffset.UTC)); + } + + /** + * Best-effort conversion of a value read from the result set into a {@link Timestamp}. + * Handles {@link Timestamp}, any {@link Date}, and parseable timestamp strings. + * Returns null when the value is null or cannot be interpreted. + */ + private static Timestamp toTimestampSafe(Object value) { + if (value == null) { + return null; + } + if (value instanceof Timestamp) { + return (Timestamp) value; + } + if (value instanceof Date) { + return new Timestamp(((Date) value).getTime()); + } + try { + return Timestamp.valueOf(value.toString().trim()); + } catch (IllegalArgumentException e) { + return null; + } + } + + private static void addPrimaryKeyIfNotExists(Connection conn, AbstractSqlQueryUtil queryUtil, + String tableName, String database, String schema, String pkName, String[] columns) { + try { + if (queryUtil.allowIfExistsAddConstraint()) { + String colList = String.join(", ", columns); + String sql = "ALTER TABLE " + tableName + " ADD CONSTRAINT IF NOT EXISTS " + + pkName + " PRIMARY KEY (" + colList + ")"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } + } else { + // Try to add and swallow the error if it already exists + String colList = String.join(", ", columns); + String sql = "ALTER TABLE " + tableName + " ADD CONSTRAINT " + + pkName + " PRIMARY KEY (" + colList + ")"; + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } + } + } catch (Exception e) { + classLogger.debug("Primary key {} may already exist on {}: {}", pkName, tableName, e.getMessage()); + } + } + + /** + * Adds a column to an existing table if it isn't already present - used to migrate + * installs that predate a column addition. Errors (column already exists) are swallowed. + */ + private static void addColumnIfNotExists(Connection conn, AbstractSqlQueryUtil queryUtil, + String tableName, String columnName, String columnType) { + try { + String sql = queryUtil.allowIfExistsModifyColumnSyntax() + ? queryUtil.alterTableAddColumnIfNotExists(tableName, columnName, columnType) + : queryUtil.alterTableAddColumn(tableName, columnName, columnType); + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } + } catch (Exception e) { + classLogger.debug("Column {} may already exist on {}: {}", columnName, tableName, e.getMessage()); + } + } + + private static void createIndexIfNotExists(Connection conn, AbstractSqlQueryUtil queryUtil, + boolean allowIfExists, String indexName, String tableName, String[] columns) { + try { + List colList = Arrays.asList(columns); + String sql; + if (allowIfExists && queryUtil.allowIfExistsIndexSyntax()) { + sql = queryUtil.createIndexIfNotExists(indexName, tableName, colList); + } else { + sql = queryUtil.createIndex(indexName, tableName, colList); + } + if (sql != null && !sql.isEmpty()) { + try (PreparedStatement ps = conn.prepareStatement(sql)) { + ps.execute(); + } + } + } catch (Exception e) { + classLogger.debug("Index {} may already exist on {}: {}", indexName, tableName, e.getMessage()); + } + } + +} diff --git a/src/prerna/reactor/automation/AutomationDefinitionValidator.java b/src/prerna/reactor/automation/AutomationDefinitionValidator.java new file mode 100644 index 00000000000..cba1b1d3761 --- /dev/null +++ b/src/prerna/reactor/automation/AutomationDefinitionValidator.java @@ -0,0 +1,342 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.PriorityQueue; +import java.util.Set; +import java.util.TreeMap; + +import com.google.gson.JsonParseException; + +import prerna.reactor.automation.nodes.IAutomationNodeExecutor; +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +/** + * Validates automation definition documents and produces a stable run provenance snapshot. + */ +public final class AutomationDefinitionValidator { + + private AutomationDefinitionValidator() {} + + /** + * Parses and validates an automation JSON document. + * + * @param json automation definition JSON + * @return validated document metadata + */ + public static ValidatedDefinition parseAndValidate(String json) { + if (json == null || json.isBlank()) { + throw new IllegalArgumentException("Automation definition must be a nonblank JSON object."); + } + try { + Map document = AutomationExecutionUtils.GSON.fromJson(json, AutomationExecutionUtils.MAP_TYPE); + return validate(document); + } catch (JsonParseException e) { + throw new IllegalArgumentException("Automation definition must be valid JSON.", e); + } + } + + /** + * Validates a parsed automation document. The returned nodes are the document's node maps, + * allowing permitted runtime overrides to be revalidated and snapshotted without reparsing. + * + * @param document parsed automation definition + * @return validated document metadata + */ + public static ValidatedDefinition validate(Map document) { + if (document == null) { + throw new IllegalArgumentException("Automation definition must be a JSON object."); + } + + int version = validateVersion(document.get(AutomationConstants.DOC_VERSION)); + Map graph = requireMap(document.get(AutomationConstants.DOC_GRAPH), "graph"); + List> nodes = requireMapList(graph.get(AutomationConstants.DOC_NODES), "graph.nodes"); + List> edges = requireMapList(graph.get(AutomationConstants.DOC_EDGES), "graph.edges"); + + Set nodeIds = validateNodes(nodes); + validateEdgesAndDag(edges, nodeIds); + + String snapshot = toCanonicalJson(document); + return new ValidatedDefinition(document, nodes, edges, version, snapshot, sha256(snapshot)); + } + + private static int validateVersion(Object value) { + if (!(value instanceof Number)) { + throw new IllegalArgumentException("Automation definition version must be " + AutomationConstants.DOC_CURRENT_VERSION + "."); + } + double number = ((Number) value).doubleValue(); + if (!Double.isFinite(number) || number != Math.rint(number) + || number != AutomationConstants.DOC_CURRENT_VERSION) { + throw new IllegalArgumentException("Unsupported automation definition version: " + value + "."); + } + return (int) number; + } + + private static Set validateNodes(List> nodes) { + Set nodeIds = new HashSet<>(); + int triggerCount = 0; + for (int i = 0; i < nodes.size(); i++) { + Map node = nodes.get(i); + String nodeId = requireNonblankString(node.get(AutomationConstants.NODE_FIELD_ID), + "graph.nodes[" + i + "].id"); + if (!nodeIds.add(nodeId)) { + throw new IllegalArgumentException("Automation definition has duplicate node id: " + nodeId + "."); + } + + String type = requireNonblankString(node.get(AutomationConstants.NODE_FIELD_TYPE), + "graph.nodes[" + i + "].type"); + if (AutomationConstants.NODE_TRIGGER.equals(type)) { + triggerCount++; + } else if (!IAutomationNodeExecutor.EXECUTORS.containsKey(type)) { + throw new IllegalArgumentException("Unsupported automation node type: " + type + "."); + } + } + if (triggerCount != 1) { + throw new IllegalArgumentException("Automation definition must contain exactly one trigger node."); + } + return nodeIds; + } + + private static void validateEdgesAndDag(List> edges, Set nodeIds) { + Map> adjacency = new HashMap<>(); + Map indegrees = new HashMap<>(); + for (String nodeId : nodeIds) { + adjacency.put(nodeId, new ArrayList<>()); + indegrees.put(nodeId, 0); + } + + for (int i = 0; i < edges.size(); i++) { + Map edge = edges.get(i); + String source = requireNonblankString(edge.get(AutomationConstants.EDGE_FIELD_SOURCE), + "graph.edges[" + i + "].source"); + String target = requireNonblankString(edge.get(AutomationConstants.EDGE_FIELD_TARGET), + "graph.edges[" + i + "].target"); + if (!nodeIds.contains(source) || !nodeIds.contains(target)) { + throw new IllegalArgumentException("Automation edge references an unknown node: " + source + " -> " + target + "."); + } + if (source.equals(target)) { + throw new IllegalArgumentException("Automation edge cannot reference the same source and target node: " + source + "."); + } + adjacency.get(source).add(target); + indegrees.put(target, indegrees.get(target) + 1); + } + + ArrayDeque ready = new ArrayDeque<>(); + for (Map.Entry entry : indegrees.entrySet()) { + if (entry.getValue() == 0) { + ready.add(entry.getKey()); + } + } + int visited = 0; + while (!ready.isEmpty()) { + String nodeId = ready.remove(); + visited++; + for (String target : adjacency.get(nodeId)) { + int remaining = indegrees.get(target) - 1; + indegrees.put(target, remaining); + if (remaining == 0) { + ready.add(target); + } + } + } + if (visited != nodeIds.size()) { + throw new IllegalArgumentException("Automation definition graph must be acyclic."); + } + } + + private static List> topologicallyOrderNodes(List> nodes, + List> edges) { + Map> nodesById = new LinkedHashMap<>(); + Map> adjacency = new LinkedHashMap<>(); + Map indegrees = new LinkedHashMap<>(); + Map nodeIndexes = new HashMap<>(); + for (int index = 0; index < nodes.size(); index++) { + Map node = nodes.get(index); + String nodeId = (String) node.get(AutomationConstants.NODE_FIELD_ID); + nodesById.put(nodeId, node); + adjacency.put(nodeId, new ArrayList<>()); + indegrees.put(nodeId, 0); + nodeIndexes.put(nodeId, index); + } + for (Map edge : edges) { + String source = (String) edge.get(AutomationConstants.EDGE_FIELD_SOURCE); + String target = (String) edge.get(AutomationConstants.EDGE_FIELD_TARGET); + adjacency.get(source).add(target); + indegrees.put(target, indegrees.get(target) + 1); + } + + PriorityQueue ready = new PriorityQueue<>((left, right) -> + Integer.compare(nodeIndexes.get(left), nodeIndexes.get(right))); + for (String nodeId : nodesById.keySet()) { + if (indegrees.get(nodeId) == 0) { + ready.add(nodeId); + } + } + + List> ordered = new ArrayList<>(); + while (!ready.isEmpty()) { + String nodeId = ready.remove(); + ordered.add(nodesById.get(nodeId)); + for (String target : adjacency.get(nodeId)) { + int remaining = indegrees.get(target) - 1; + indegrees.put(target, remaining); + if (remaining == 0) { + ready.add(target); + } + } + } + return ordered; + } + + private static Map requireMap(Object value, String field) { + if (!(value instanceof Map)) { + throw new IllegalArgumentException("Automation definition field '" + field + "' must be an object."); + } + for (Map.Entry entry : ((Map) value).entrySet()) { + if (!(entry.getKey() instanceof String)) { + throw new IllegalArgumentException("Automation definition field '" + field + "' has a non-string key."); + } + } + @SuppressWarnings("unchecked") + Map source = (Map) value; + return source; + } + + private static List> requireMapList(Object value, String field) { + if (!(value instanceof List)) { + throw new IllegalArgumentException("Automation definition field '" + field + "' must be an array."); + } + List> maps = new ArrayList<>(); + List values = (List) value; + for (int i = 0; i < values.size(); i++) { + maps.add(requireMap(values.get(i), field + "[" + i + "]")); + } + return maps; + } + + private static String requireNonblankString(Object value, String field) { + if (!(value instanceof String) || ((String) value).isBlank()) { + throw new IllegalArgumentException("Automation definition field '" + field + "' must be a nonblank string."); + } + return (String) value; + } + + private static String toCanonicalJson(Map document) { + return AutomationExecutionUtils.GSON.toJson(canonicalize(document)); + } + + private static Object canonicalize(Object value) { + if (value instanceof Map) { + Map sorted = new TreeMap<>(); + for (Map.Entry entry : ((Map) value).entrySet()) { + if (!(entry.getKey() instanceof String)) { + throw new IllegalArgumentException("Automation definition contains a non-string object key."); + } + sorted.put((String) entry.getKey(), canonicalize(entry.getValue())); + } + return sorted; + } + if (value instanceof List) { + List values = new ArrayList<>(); + for (Object element : (List) value) { + values.add(canonicalize(element)); + } + return values; + } + return value; + } + + private static String sha256(String value) { + try { + byte[] hash = MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); + return java.util.HexFormat.of().formatHex(hash); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable.", e); + } + } + + /** + * Immutable metadata for a validated document and its canonical execution snapshot. + */ + public static final class ValidatedDefinition { + + private final Map document; + private final List> nodes; + private final List> edges; + private final int version; + private final String snapshot; + private final String hash; + + private ValidatedDefinition(Map document, List> nodes, + List> edges, + int version, String snapshot, String hash) { + this.document = document; + this.nodes = nodes; + this.edges = edges; + this.version = version; + this.snapshot = snapshot; + this.hash = hash; + } + + public Map getDocument() { + return document; + } + + public List> getNodes() { + return nodes; + } + + /** + * Returns nodes in dependency order. Nodes that become ready together retain their + * persisted document order, keeping shared-scope execution deterministic. + */ + public List> getExecutionOrder() { + return topologicallyOrderNodes(nodes, edges); + } + + public int getVersion() { + return version; + } + + public String getSnapshot() { + return snapshot; + } + + public String getHash() { + return hash; + } + } +} diff --git a/src/prerna/reactor/automation/AutomationMcpSync.java b/src/prerna/reactor/automation/AutomationMcpSync.java new file mode 100644 index 00000000000..3830e43e412 --- /dev/null +++ b/src/prerna/reactor/automation/AutomationMcpSync.java @@ -0,0 +1,393 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; + +import prerna.auth.User; +import prerna.cluster.util.ClusterUtil; +import prerna.project.api.IProject; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.reactor.agent.mcp.MCPUtility.MCPDisplayOption; +import prerna.reactor.agent.mcp.MCPUtility.MCPExecution; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.util.AssetUtility; +import prerna.util.Constants; +import prerna.util.git.GitRepoUtils; + +/** + * Keeps each project's own {@code assets/mcp/pixel_mcp.json} in sync with a project-scoped + * {@code TriggerAutomation} MCP tool entry, called by {@link SaveAutomationReactor} on every save + * so the project's automation is always discoverable as an MCP tool without a separate manual + * "make this an MCP tool" step. + * + *

A single-purpose class (not folded into {@link AutomationExecutionUtils}, which is scoped to + * run-execution concerns) so the MCP-catalog-sync responsibility stays isolated and easy to find. + * + *

Uses {@code org.json} (JSONObject/JSONArray) throughout because {@link MCPUtility} is built + * on org.json, and converting between org.json and Gson just to call those helpers would add + * unnecessary overhead and type-safety risk. Gson is used everywhere else in the automation package. + */ +public final class AutomationMcpSync { + + private static final Logger classLogger = LogManager.getLogger(AutomationMcpSync.class); + + /** Stamped into the generated tool as {@link MCPUtility#SMSS_MCP_GENERATOR} so re-saves replace it in place. */ + private static final String AUTOMATION_MCP_GENERATOR_ID = "AutomationMCP"; + + private AutomationMcpSync() { + // utility class + } + + /** + * Writes/updates the project-scoped {@code TriggerAutomation} entry. Uses the same + * merge/generator-stamp helpers as {@code MakePixelMCPReactor} (a distinct generator id), so + * it never disturbs tools the user authored by hand or generated through other flows in the + * same file. + * + *

{@code project} is kept as a required argument on the generated tool (matching the + * existing convention for reactor-scanned MCP tools) rather than hardcoded - the id is still + * fixed to this project by construction of the pixel expression itself, so callers only ever + * need to (re-)confirm which project, never guess a different one. + * + *

Failures here are logged and swallowed - the automation save itself must not fail just + * because the MCP catalog couldn't be refreshed. + * + * @param project the resolved project, or {@code null} if it could not be loaded (e.g. + * deleted concurrently) - a no-op in that case, logged as a warning + * @param projectId the project id (used even when {@code project} is present, for clarity) + * @param user the user performing the save, used as the git commit author + */ + public static void syncTriggerAutomationTool(IProject project, String projectId, User user, String automationJson) { + if (project == null) { + classLogger.warn("Skipping automation MCP tool sync for project {}: project could not be loaded.", + projectId); + return; + } + if (automationJson == null || automationJson.isBlank()) { + classLogger.warn("Skipping automation MCP tool sync for project {}: automation JSON is empty.", projectId); + return; + } + + try { + boolean hasDbNodes = hasPlaygroundDbNodes(automationJson); + JSONArray generated = new JSONArray().put(buildTriggerAutomationTool(projectId, automationJson, hasDbNodes)); + if (hasDbNodes) { + generated.put(buildGetAutomationSchemaTool(projectId)); + } + generated.put(buildBuildAutomationTool(projectId)); + MCPUtility.stampGenerator(generated, AUTOMATION_MCP_GENERATOR_ID); + + // One-arg form is equivalent to two-arg: it does Utility.getProject(projectId) internally + // and then delegates to getProjectAssetsFolder(projectName, projectId) - same path, no difference. + // We already hold IProject above but the registry lookup is cheap and avoids coupling the + // call site to projectName extraction just for this one call. + String assetsFolder = AssetUtility.getProjectAssetsFolder(projectId); + String outputFileLoc = Paths.get(assetsFolder, "mcp", "pixel_mcp.json").toString(); + JSONArray merged = MCPUtility.mergeGeneratedTools( + MCPUtility.readMcpJson(outputFileLoc), generated, AUTOMATION_MCP_GENERATOR_ID, true); + + writeMcpJson(outputFileLoc, merged); + + MCPUtility.addMCPTag(project); + commitAndPush(project, projectId, assetsFolder, user); + } catch (Exception e) { + classLogger.warn("Failed to sync TriggerAutomation MCP tool for project {}", projectId, e); + } + } + + // -- Private helpers ------------------------------------------------------------- + + private static JSONObject buildTriggerAutomationTool(String projectId, String automationJson, boolean hasDbNodes) { + JSONObject tool = new JSONObject(); + tool.put("name", "TriggerAutomation"); + tool.put("title", "Trigger Automation"); + + String docDescription = null; + try { + if (automationJson != null && !automationJson.isBlank()) { + JSONObject doc = new JSONObject(automationJson); + String raw = doc.optString(AutomationConstants.DOC_DESCRIPTION, "").trim(); + if (!raw.isEmpty()) { + docDescription = raw; + } + } + } catch (Exception e) { + classLogger.warn("Failed to read description from automation JSON for project {}", projectId, e); + } + + String description; + if (docDescription != null) { + description = docDescription + " Triggers the automation and returns a per-workflow summary once complete."; + } else { + description = "Manually triggers the automation configured for this project/app and returns a " + + "per-workflow summary once complete (e.g. \"Indexed 20 files\")."; + } + if (hasDbNodes) { + description += " This automation has database nodes that accept SQL queries - call GetAutomationSchema first" + + " to discover the exact table and column names before writing SQL."; + } + tool.put("description", description); + + JSONObject projectProp = new JSONObject(); + projectProp.put("type", "string"); + projectProp.put("title", "Project"); + projectProp.put("description", "The project ID for this automation. Always use: " + projectId); + projectProp.put("default", projectId); + JSONObject properties = new JSONObject(); + properties.put(ReactorKeysEnum.PROJECT.getKey(), projectProp); + + JSONObject inputsProperties = new JSONObject(); + try { + if (automationJson != null && !automationJson.isBlank()) { + JSONObject doc = new JSONObject(automationJson); + JSONObject graph = doc.optJSONObject("graph"); + JSONArray nodes = graph != null ? graph.optJSONArray("nodes") : null; + if (nodes != null) { + for (int i = 0; i < nodes.length(); i++) { + JSONObject node = nodes.optJSONObject(i); + if (node == null) continue; + String nodeLabel = node.optString("label", ""); + JSONArray fillable = node.optJSONArray("playgroundFillable"); + if (fillable == null || fillable.length() == 0) continue; + String nodeType = node.optString("type", ""); + for (int j = 0; j < fillable.length(); j++) { + String fieldName = fillable.optString(j); + if (fieldName == null || fieldName.isBlank()) continue; + String paramName = AutomationExecutionUtils.buildPlaygroundParamName(nodeLabel, fieldName); + String paramDescription = buildPlaygroundParamDescription(nodeType, fieldName); + JSONObject paramProp = new JSONObject(); + paramProp.put("type", "string"); + paramProp.put("description", paramDescription); + inputsProperties.put(paramName, paramProp); + } + } + } + } + } catch (Exception e) { + classLogger.warn("Failed to scan automation nodes for playground inputs for project {}", projectId, e); + } + + if (!inputsProperties.isEmpty()) { + JSONObject inputsProp = new JSONObject(); + inputsProp.put("type", "object"); + inputsProp.put("description", "Optional inputs to inject into automation nodes before running. Populate fields with values relevant to the user's request."); + inputsProp.put("properties", inputsProperties); + properties.put(AutomationConstants.AUTOMATION_INPUTS_KEY, inputsProp); + } + + JSONObject triggerTypeProp = new JSONObject(); + triggerTypeProp.put("type", "string"); + triggerTypeProp.put("title", "Trigger Type"); + triggerTypeProp.put("description", "How this automation was triggered. Always use: " + AutomationConstants.TRIGGER_PLAYGROUND); + triggerTypeProp.put("default", AutomationConstants.TRIGGER_PLAYGROUND); + properties.put(AutomationConstants.AUTOMATION_TRIGGER_TYPE_KEY, triggerTypeProp); + + JSONObject inputSchema = new JSONObject(); + inputSchema.put("type", "object"); + inputSchema.put("title", "TriggerAutomation_Arguments"); + inputSchema.put("properties", properties); + inputSchema.put("required", new JSONArray().put(ReactorKeysEnum.PROJECT.getKey())); + tool.put("inputSchema", inputSchema); + + JSONObject uiJson = new JSONObject(); + uiJson.put(MCPUtility.UI_DISPLAY_LOCATION, MCPDisplayOption.SIDEBAR.getValue()); + uiJson.put(MCPUtility.UI_RESOURCE_URI, "system://automation-workspace/?readOnly=1"); + + JSONObject meta = new JSONObject(); + meta.put(MCPUtility.SMSS_FUNCTION_NAME, "TriggerAutomation"); + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPExecution.ASK.getValue()); + meta.put(MCPUtility.SMSS_MCP_UI, uiJson); + tool.put("_meta", meta); + + return tool; + } + + /** Returns true if any database-engine node has {@code expression} in its {@code playgroundFillable} list. */ + private static boolean hasPlaygroundDbNodes(String automationJson) { + if (automationJson == null || automationJson.isBlank()) return false; + try { + JSONObject doc = new JSONObject(automationJson); + JSONObject graph = doc.optJSONObject("graph"); + JSONArray nodes = graph != null ? graph.optJSONArray("nodes") : null; + if (nodes == null) return false; + for (int i = 0; i < nodes.length(); i++) { + JSONObject node = nodes.optJSONObject(i); + if (node == null) continue; + if (!AutomationConstants.NODE_DATABASE_ENGINE.equals(node.optString("type"))) continue; + JSONArray fillable = node.optJSONArray("playgroundFillable"); + if (fillable == null) continue; + for (int j = 0; j < fillable.length(); j++) { + if (AutomationConstants.CONFIG_EXPRESSION.equals(fillable.optString(j))) return true; + } + } + } catch (Exception e) { + classLogger.warn("Failed to parse automation JSON while checking for DB nodes", e); + } + return false; + } + + /** + * Builds the {@code BuildAutomation} MCP tool - lets an agent in Playground generate or edit + * this project's automation from a plain-English description. Execution mode is ASK so the user + * can review the generated document before it is saved. + */ + private static JSONObject buildBuildAutomationTool(String projectId) { + JSONObject tool = new JSONObject(); + tool.put("name", "BuildAutomation"); + tool.put("title", "Build / Edit Automation"); + tool.put("description", + "Generates or edits this project's automation from a plain-English description. " + + "The model iteratively gathers context (database schema, available reactors) before producing a complete automation document. " + + "Does NOT save automatically - call SaveAutomation with the returned JSON to persist. " + + "Use currentDoc to pass the existing automation JSON (base64-encoded) for edit mode."); + + JSONObject projectProp = new JSONObject(); + projectProp.put("type", "string"); + projectProp.put("description", "The project ID for this automation. Always use: " + projectId); + projectProp.put("default", projectId); + + JSONObject descProp = new JSONObject(); + descProp.put("type", "string"); + descProp.put("description", "Plain-English description of what the automation should do, or how to modify the existing one. Will be base64-encoded automatically if needed."); + + JSONObject currentDocProp = new JSONObject(); + currentDocProp.put("type", "string"); + currentDocProp.put("description", "Optional base64-encoded JSON of the current automation document. Include this to edit rather than generate from scratch."); + + JSONObject properties = new JSONObject(); + properties.put(ReactorKeysEnum.PROJECT.getKey(), projectProp); + properties.put(AutomationConstants.DOC_DESCRIPTION, descProp); + properties.put("currentDoc", currentDocProp); + + JSONObject inputSchema = new JSONObject(); + inputSchema.put("type", "object"); + inputSchema.put("title", "BuildAutomation_Arguments"); + inputSchema.put("properties", properties); + inputSchema.put("required", new JSONArray().put(ReactorKeysEnum.PROJECT.getKey()).put(AutomationConstants.DOC_DESCRIPTION)); + tool.put("inputSchema", inputSchema); + + JSONObject meta = new JSONObject(); + meta.put(MCPUtility.SMSS_FUNCTION_NAME, "BuildAutomation"); + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPExecution.ASK.getValue()); + tool.put("_meta", meta); + + return tool; + } + + /** Builds the auto-executable {@code GetAutomationSchema} companion tool. */ + private static JSONObject buildGetAutomationSchemaTool(String projectId) { + JSONObject tool = new JSONObject(); + tool.put("name", "GetAutomationSchema"); + tool.put("title", "Get Automation Database Schema"); + tool.put("description", + "Returns the physical table and column names for database nodes in this automation that accept SQL input. " + + "Call this before TriggerAutomation when you need to write a SQL query - it gives you the exact " + + "table and column names available in the database."); + + JSONObject projectProp = new JSONObject(); + projectProp.put("type", "string"); + projectProp.put("description", "The project ID for this automation. Always use: " + projectId); + projectProp.put("default", projectId); + JSONObject properties = new JSONObject(); + properties.put(ReactorKeysEnum.PROJECT.getKey(), projectProp); + + JSONObject inputSchema = new JSONObject(); + inputSchema.put("type", "object"); + inputSchema.put("title", "GetAutomationSchema_Arguments"); + inputSchema.put("properties", properties); + inputSchema.put("required", new JSONArray().put(ReactorKeysEnum.PROJECT.getKey())); + tool.put("inputSchema", inputSchema); + + JSONObject meta = new JSONObject(); + meta.put(MCPUtility.SMSS_FUNCTION_NAME, "GetAutomationSchema"); + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPExecution.AUTO.getValue()); + tool.put("_meta", meta); + + return tool; + } + + private static void writeMcpJson(String outputFileLoc, JSONArray tools) throws IOException { + JSONObject mcpJson = new JSONObject(); + mcpJson.put("tools", tools); + JSONObject fileMeta = new JSONObject(); + fileMeta.put("last_modified_date", LocalDate.now(ZoneOffset.UTC).format(DateTimeFormatter.ISO_LOCAL_DATE)); + mcpJson.put("_meta", fileMeta); + + File outputFile = new File(outputFileLoc); + outputFile.getParentFile().mkdirs(); + Files.writeString(outputFile.toPath(), mcpJson.toString(4), StandardCharsets.UTF_8); + } + + private static String buildPlaygroundParamDescription(String nodeType, String fieldName) { + if (AutomationConstants.NODE_DATABASE_ENGINE.equals(nodeType) && AutomationConstants.CONFIG_EXPRESSION.equals(fieldName)) { + return "SQL query to execute against the connected database"; + } + if (AutomationConstants.NODE_MODEL_ENGINE.equals(nodeType) && AutomationConstants.CONFIG_COMMAND.equals(fieldName)) { + return "Natural language prompt to send to the language model"; + } + if (AutomationConstants.NODE_MODEL_ENGINE.equals(nodeType) && AutomationConstants.CONFIG_CONTEXT.equals(fieldName)) { + return "System instructions for the language model's behavior"; + } + if (AutomationConstants.NODE_VECTOR_ENGINE.equals(nodeType) && AutomationConstants.CONFIG_COMMAND.equals(fieldName)) { + return "Search query to run against the vector database"; + } + if (AutomationConstants.NODE_FUNCTION_ENGINE.equals(nodeType) && AutomationConstants.CONFIG_PARAMS.equals(fieldName)) { + return "JSON parameters to pass to the function"; + } + return "Input for the " + fieldName + " field"; + } + + private static void commitAndPush(IProject project, String projectId, String assetsFolder, User user) { + String versionFolder = AssetUtility.getProjectVersionFolder(project.getProjectName(), projectId); + List gitRelativeFilePaths = new ArrayList<>(); + gitRelativeFilePaths.add(Constants.ASSETS_FOLDER + "/mcp/pixel_mcp.json"); + GitRepoUtils.addSpecificFiles(versionFolder, gitRelativeFilePaths); + GitRepoUtils.commitAddedFiles(versionFolder, "sync: automation MCP tool", user); + + if (ClusterUtil.IS_CLUSTER) { + ClusterUtil.pushProjectFolder(project, assetsFolder); + } + } +} diff --git a/src/prerna/reactor/automation/AutomationOwlCreator.java b/src/prerna/reactor/automation/AutomationOwlCreator.java new file mode 100644 index 00000000000..740e009b5f0 --- /dev/null +++ b/src/prerna/reactor/automation/AutomationOwlCreator.java @@ -0,0 +1,138 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import static prerna.reactor.automation.AutomationConstants.AUTOMATION_ID; +import static prerna.reactor.automation.AutomationConstants.BIGINT; +import static prerna.reactor.automation.AutomationConstants.CANCEL_REQUESTED; +import static prerna.reactor.automation.AutomationConstants.CLAIMED_AT; +import static prerna.reactor.automation.AutomationConstants.COMPLETED_AT; +import static prerna.reactor.automation.AutomationConstants.COMPLETED_NODES; +import static prerna.reactor.automation.AutomationConstants.CREATED_BY; +import static prerna.reactor.automation.AutomationConstants.DEFINITION_HASH; +import static prerna.reactor.automation.AutomationConstants.DEFINITION_SNAPSHOT; +import static prerna.reactor.automation.AutomationConstants.DEFINITION_VERSION; +import static prerna.reactor.automation.AutomationConstants.DURATION_MS; +import static prerna.reactor.automation.AutomationConstants.ERROR_MESSAGE; +import static prerna.reactor.automation.AutomationConstants.EXECUTION_ORDER; +import static prerna.reactor.automation.AutomationConstants.FAILED_NODE_ID; +import static prerna.reactor.automation.AutomationConstants.INTEGER; +import static prerna.reactor.automation.AutomationConstants.LAST_HEARTBEAT; +import static prerna.reactor.automation.AutomationConstants.NODE_ID; +import static prerna.reactor.automation.AutomationConstants.NODE_LABEL; +import static prerna.reactor.automation.AutomationConstants.OUTPUT_PREVIEW; +import static prerna.reactor.automation.AutomationConstants.OUTPUT_VALUE; +import static prerna.reactor.automation.AutomationConstants.OUTPUT_VAR; +import static prerna.reactor.automation.AutomationConstants.PROJECT_ID; +import static prerna.reactor.automation.AutomationConstants.RESULT_SUMMARY_COL; +import static prerna.reactor.automation.AutomationConstants.RUN_ID; +import static prerna.reactor.automation.AutomationConstants.STARTED_AT; +import static prerna.reactor.automation.AutomationConstants.STATUS; +import static prerna.reactor.automation.AutomationConstants.TABLE_AUTOMATION_ACTIVE_RUN; +import static prerna.reactor.automation.AutomationConstants.TABLE_AUTOMATION_NODE_OUTPUTS; +import static prerna.reactor.automation.AutomationConstants.TABLE_AUTOMATION_RUNS; +import static prerna.reactor.automation.AutomationConstants.TOTAL_NODES; +import static prerna.reactor.automation.AutomationConstants.TRIGGER_TYPE; +import static prerna.reactor.automation.AutomationConstants.VARCHAR_2000; +import static prerna.reactor.automation.AutomationConstants.VARCHAR_255; +import static prerna.reactor.automation.AutomationConstants.VARCHAR_500; + +import java.util.ArrayList; +import java.util.Arrays; + +import org.javatuples.Pair; + +import prerna.engine.impl.owl.AbstractOwlCreator; + +/** + * OWL schema declaration for the three automation tables stored in the + * scheduler database. Follows the same pattern as other system-engine OWL + * creators (e.g. {@link prerna.reactor.scheduler.SchedulerOwlCreator}). + * + *

Called from {@link AutomationDatabaseUtility#initialize()} so that the + * automation OWL schema stays entirely within the {@code prerna.reactor.automation} + * package and does not require the scheduler package to import automation types. + */ +public class AutomationOwlCreator extends AbstractOwlCreator { + + // Reuse the DB-level BOOLEAN/TIMESTAMP/CLOB type names used by the scheduler OWL. + // SchedulerConstants uses these string literals directly (e.g. "BOOLEAN", "TIMESTAMP", "CLOB"). + private static final String BOOLEAN = "BOOLEAN"; + private static final String TIMESTAMP = "TIMESTAMP"; + private static final String CLOB = "CLOB"; + + public AutomationOwlCreator() { + createColumnsAndTypes(); + } + + public void createColumnsAndTypes() { + this.allSchemas = new ArrayList<>(); + + // @formatter:off + addTable(TABLE_AUTOMATION_RUNS, Arrays.asList( + Pair.with(RUN_ID, VARCHAR_255), + Pair.with(PROJECT_ID, VARCHAR_255), + Pair.with(AUTOMATION_ID, VARCHAR_255), + Pair.with(DEFINITION_VERSION, INTEGER), + Pair.with(DEFINITION_HASH, VARCHAR_255), + Pair.with(DEFINITION_SNAPSHOT, CLOB), + Pair.with(STATUS, VARCHAR_255), + Pair.with(TRIGGER_TYPE, VARCHAR_255), + Pair.with(STARTED_AT, TIMESTAMP), + Pair.with(COMPLETED_AT, TIMESTAMP), + Pair.with(FAILED_NODE_ID, VARCHAR_255), + Pair.with(ERROR_MESSAGE, CLOB), + Pair.with(LAST_HEARTBEAT, TIMESTAMP), + Pair.with(TOTAL_NODES, INTEGER), + Pair.with(COMPLETED_NODES, INTEGER), + Pair.with(CREATED_BY, VARCHAR_255), + Pair.with(CANCEL_REQUESTED, BOOLEAN), + Pair.with(RESULT_SUMMARY_COL, VARCHAR_2000))); + + addTable(TABLE_AUTOMATION_NODE_OUTPUTS, Arrays.asList( + Pair.with(RUN_ID, VARCHAR_255), + Pair.with(NODE_ID, VARCHAR_255), + Pair.with(NODE_LABEL, VARCHAR_500), + Pair.with(EXECUTION_ORDER, INTEGER), + Pair.with(STATUS, VARCHAR_255), + Pair.with(STARTED_AT, TIMESTAMP), + Pair.with(COMPLETED_AT, TIMESTAMP), + Pair.with(DURATION_MS, BIGINT), + Pair.with(OUTPUT_VAR, VARCHAR_255), + Pair.with(OUTPUT_VALUE, CLOB), + Pair.with(OUTPUT_PREVIEW, VARCHAR_2000), + Pair.with(ERROR_MESSAGE, CLOB))); + + addTable(TABLE_AUTOMATION_ACTIVE_RUN, Arrays.asList( + Pair.with(PROJECT_ID, VARCHAR_255), + Pair.with(RUN_ID, VARCHAR_255), + Pair.with(CLAIMED_AT, TIMESTAMP))); + // @formatter:on + } + +} diff --git a/src/prerna/reactor/automation/AutomationRunEngine.java b/src/prerna/reactor/automation/AutomationRunEngine.java new file mode 100644 index 00000000000..5ad3efad169 --- /dev/null +++ b/src/prerna/reactor/automation/AutomationRunEngine.java @@ -0,0 +1,309 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.PixelExecutionUtils; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +import java.sql.Timestamp; +import java.time.Instant; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.om.Insight; +import prerna.om.ThreadStore; +import prerna.reactor.automation.nodes.AutomationNodeContext; +import prerna.reactor.automation.nodes.IAutomationNodeExecutor; +import prerna.sablecc2.comm.PixelJobManager; +import prerna.util.Utility; + +/** + * Executes an automation run synchronously. Called by {@link TriggerAutomationReactor} + * on the virtual thread provided by the platform's {@code runPixelAsync} endpoint. + * Iterates the dependency-ordered nodes supplied by the validated definition, dispatches each to its {@link IAutomationNodeExecutor}, + * and writes per-node status to the DB as it goes. + */ +public final class AutomationRunEngine { + + private static final Logger classLogger = LogManager.getLogger(AutomationRunEngine.class); + + /** In-memory cancellation flags - fast path; the DB flag is the cluster-safe source of truth. */ + static final ConcurrentHashMap CANCELLATION_FLAGS = new ConcurrentHashMap<>(); + + private AutomationRunEngine() {} + + /** + * In-memory same-pod fast path for cancellation. + * Called by {@link CancelAutomationRunReactor}. + */ + public static boolean requestCancellation(String runId) { + AtomicBoolean flag = CANCELLATION_FLAGS.get(runId); + if (flag != null) { + flag.set(true); + return true; + } + return false; + } + + /** + * Runs the full automation node list, blocking until all nodes complete or the run is + * cancelled/failed. Runs synchronously on the calling virtual thread. + * + * @param runId the run record ID (already inserted into DB by the caller) + * @param projectId the owning project + * @param ordered nodes in execution order + * @param configMap project automation config key-value pairs + * @param insight the caller's insight context (propagated to each node executor) + * @return the run's final variable scope (trigger vars + every node's {@code outputVar} + * that completed successfully) - used by the caller to resolve a per-workflow + * summary message once the run finishes + */ + public static Map run(String runId, String projectId, + List> ordered, Map configMap, Insight insight) { + + AtomicBoolean cancelled = new AtomicBoolean(false); + CANCELLATION_FLAGS.put(runId, cancelled); + ScheduledExecutorService heartbeat = startHeartbeat(runId); + + // Captured once here - TriggerAutomationReactor runs on the virtual thread the platform's + // runPixelAsync endpoint spawns, so ThreadStore carries that job's id for the whole call. + // Used to stream per-node progress the same way HarnessToolExecutor streams tool-call + // progress during an agent turn (see PixelJobManager#addStreamOut), so the FE can poll + // getPixelJobStreaming(jobId) for live node status instead of inferring it from DB polls. + String jobId = ThreadStore.getJobId(); + + Map scope = AutomationExecutionUtils.buildInitialScope(runId, insight.getUser()); + int completedCount = 0; + + try { + for (Map node : ordered) { + String nodeId = (String) node.get(AutomationConstants.NODE_FIELD_ID); + String nodeLabel = (String) node.get(AutomationConstants.NODE_FIELD_LABEL); + String outputVar = (String) node.get(AutomationConstants.NODE_FIELD_OUTPUT_VAR); + String nodeType = (String) node.get(AutomationConstants.NODE_FIELD_TYPE); + + if (cancelled.get() || AutomationDatabaseUtility.isCancelRequested(runId)) { + classLogger.info("Automation run {} cancelled before node {} ({})", runId, nodeId, nodeLabel); + AutomationDatabaseUtility.skipPendingNodes(runId, "Run cancelled by user"); + AutomationDatabaseUtility.updateRunStatus(runId, + AutomationConstants.STATUS_CANCELLED, nodeId, "Run cancelled by user"); + return scope; + } + + publishNodeEvent(jobId, nodeId, nodeLabel, AutomationConstants.NODE_STATUS_RUNNING, null, null, null); + + Map nodeResult; + try { + nodeResult = executeSingleNode(runId, projectId, node, scope, configMap, cancelled, insight); + } catch (PixelExecutionUtils.AutomationCancelledException ace) { + classLogger.info("Automation run {} cancelled during node {} ({})", runId, nodeId, nodeLabel); + AutomationDatabaseUtility.skipPendingNodes(runId, "Run cancelled by user"); + AutomationDatabaseUtility.updateRunStatus(runId, + AutomationConstants.STATUS_CANCELLED, nodeId, ace.getMessage()); + publishNodeEvent(jobId, nodeId, nodeLabel, AutomationConstants.STATUS_CANCELLED, null, null, + ace.getMessage()); + return scope; + } + + String status = (String) nodeResult.get(AutomationConstants.STATUS); + Object durationMs = nodeResult.get(AutomationConstants.DURATION_MS); + String preview = (String) nodeResult.get(AutomationConstants.OUTPUT_PREVIEW); + String errorMsg = (String) nodeResult.get(AutomationConstants.ERROR_MESSAGE); + publishNodeEvent(jobId, nodeId, nodeLabel, status, durationMs, preview, errorMsg); + + if (AutomationConstants.NODE_STATUS_SUCCESS.equals(status)) { + if (outputVar != null && !outputVar.isEmpty() + && !AutomationConstants.NODE_TRIGGER.equals(nodeType)) { + String outputValue = (String) nodeResult.get(AutomationConstants.RESULT_OUTPUT_VALUE); + scope.put(outputVar, outputValue != null ? outputValue : ""); + } + completedCount++; + AutomationDatabaseUtility.updateHeartbeat(runId, completedCount); + } else { + classLogger.warn("Automation run {} failed at node {} ({}): {}", runId, nodeId, nodeLabel, errorMsg); + AutomationDatabaseUtility.skipPendingNodes(runId, "Skipped because an earlier node failed"); + AutomationDatabaseUtility.updateRunStatus(runId, + AutomationConstants.STATUS_FAILED, nodeId, errorMsg); + return scope; + } + } + + classLogger.info("Automation run {} completed successfully ({}/{} nodes)", runId, completedCount, ordered.size()); + AutomationDatabaseUtility.updateRunStatus(runId, AutomationConstants.STATUS_SUCCESS, null, null); + return scope; + + } finally { + heartbeat.shutdownNow(); + CANCELLATION_FLAGS.remove(runId); + AutomationDatabaseUtility.releaseActiveRun(projectId, runId); + } + } + + // -- Streaming ------------------------------------------------------------------- + + /** + * Publishes a per-node progress event onto the pixel job's stream, mirroring + * {@code HarnessToolExecutor.publishToolResult} - the FE polls {@code getPixelJobStreaming(jobId)} + * (the same mechanism playground uses for live tool-call progress) to render each node's + * running/success/failed transition as it happens, instead of inferring progress from DB polls. + * A no-op when {@code jobId} is blank (e.g. called outside a {@code runPixelAsync} job). + */ + private static void publishNodeEvent(String jobId, String nodeId, String nodeLabel, String status, + Object durationMs, String preview, String errorMessage) { + if (jobId == null || jobId.isBlank()) { + return; + } + Map data = new LinkedHashMap<>(); + data.put("kind", "node-status"); + data.put(AutomationConstants.NODE_ID, nodeId); + data.put(AutomationConstants.NODE_LABEL, nodeLabel); + data.put(AutomationConstants.STATUS, status); + if (durationMs != null) { + data.put(AutomationConstants.DURATION_MS, durationMs); + } + if (preview != null) { + data.put(AutomationConstants.OUTPUT_PREVIEW, preview); + } + if (errorMessage != null) { + data.put(AutomationConstants.ERROR_MESSAGE, errorMessage); + } + data.put("timestamp", Instant.now().toString()); + + Map envelope = new LinkedHashMap<>(); + envelope.put("stream_type", "automation"); + envelope.put("data", data); + PixelJobManager.getManager().addStreamOut(jobId, envelope); + } + + // -- Node execution ------------------------------------------------------------ + + private static Map executeSingleNode(String runId, String projectId, + Map node, Map scope, Map configMap, + AtomicBoolean cancelFlag, Insight insight) { + + String nodeId = (String) node.get(AutomationConstants.NODE_FIELD_ID); + String nodeLabel = (String) node.get(AutomationConstants.NODE_FIELD_LABEL); + String outputVar = (String) node.get(AutomationConstants.NODE_FIELD_OUTPUT_VAR); + String type = (String) node.get(AutomationConstants.NODE_FIELD_TYPE); + + classLogger.debug("Executing node {} ({}) type={} in run {}", nodeId, nodeLabel, type, runId); + AutomationDatabaseUtility.markNodeRunning(runId, nodeId); + Timestamp startedAt = Utility.getSqlTimestampUTC(java.time.LocalDateTime.ofInstant(Instant.now(), java.time.ZoneOffset.UTC)); + long startMs = System.currentTimeMillis(); + + try { + if (AutomationConstants.NODE_TRIGGER.equals(type)) { + String triggeredAt = scope.get(AutomationConstants.SCOPE_TRIGGERED_AT); + AutomationDatabaseUtility.updateNodeSuccess(runId, nodeId, startedAt, 0, outputVar, triggeredAt, triggeredAt); + Map result = buildNodeResult(nodeId, nodeLabel, + AutomationConstants.NODE_STATUS_SUCCESS, 0, triggeredAt, null); + result.put(AutomationConstants.RESULT_OUTPUT_VALUE, triggeredAt); + return result; + } + + AutomationNodeContext ctx = new AutomationNodeContext( + runId, projectId, node, scope, configMap, insight, cancelFlag); + IAutomationNodeExecutor executor = IAutomationNodeExecutor.EXECUTORS.get(type); + if (executor == null) { + throw new IllegalArgumentException("Unsupported node type: " + type); + } + Object rawOutput = executor.execute(ctx); + + @SuppressWarnings("unchecked") + Map transformConfig = (Map) node.get(AutomationConstants.NODE_FIELD_OUTPUT_TRANSFORM); + String transformed = AutomationExecutionUtils.applyOutputTransform(rawOutput, transformConfig); + long durationMs = System.currentTimeMillis() - startMs; + String preview = AutomationExecutionUtils.generatePreview(transformed); + + AutomationDatabaseUtility.updateNodeSuccess(runId, nodeId, startedAt, durationMs, outputVar, transformed, preview); + + classLogger.debug("Node {} ({}) succeeded in {}ms in run {}", nodeId, nodeLabel, durationMs, runId); + Map result = buildNodeResult(nodeId, nodeLabel, + AutomationConstants.NODE_STATUS_SUCCESS, durationMs, transformed, null); + result.put(AutomationConstants.RESULT_OUTPUT_VALUE, transformed); + return result; + + } catch (PixelExecutionUtils.AutomationCancelledException ace) { + long durationMs = System.currentTimeMillis() - startMs; + AutomationDatabaseUtility.updateNodeFailed(runId, nodeId, startedAt, durationMs, ace.getMessage()); + throw ace; + } catch (Exception e) { + long durationMs = System.currentTimeMillis() - startMs; + String errorMsg = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); + classLogger.error("Node {} ({}) failed in run {}: {}", nodeId, nodeLabel, runId, errorMsg, e); + AutomationDatabaseUtility.updateNodeFailed(runId, nodeId, startedAt, durationMs, errorMsg); + return buildNodeResult(nodeId, nodeLabel, + AutomationConstants.NODE_STATUS_FAILED, durationMs, null, errorMsg); + } + } + + // -- Heartbeat ----------------------------------------------------------------- + + private static ScheduledExecutorService startHeartbeat(String runId) { + ScheduledExecutorService scheduler = Executors.newSingleThreadScheduledExecutor(r -> { + Thread t = new Thread(r, "automation-heartbeat-" + runId.substring(0, Math.min(8, runId.length()))); + t.setDaemon(true); + return t; + }); + scheduler.scheduleAtFixedRate(() -> { + try { + AutomationDatabaseUtility.touchHeartbeat(runId); + } catch (Exception e) { + classLogger.warn("Heartbeat update failed for run {}: {}", runId, e.getMessage()); + } + }, AutomationConstants.HEARTBEAT_INTERVAL_SECONDS, + AutomationConstants.HEARTBEAT_INTERVAL_SECONDS, TimeUnit.SECONDS); + return scheduler; + } + + // -- Helpers ------------------------------------------------------------------- + + private static Map buildNodeResult(String nodeId, String nodeLabel, + String status, long durationMs, String outputPreview, String errorMessage) { + Map result = new HashMap<>(); + result.put(AutomationConstants.NODE_ID, nodeId); + result.put(AutomationConstants.NODE_LABEL, nodeLabel); + result.put(AutomationConstants.STATUS, status); + result.put(AutomationConstants.DURATION_MS, durationMs); + if (outputPreview != null) result.put(AutomationConstants.OUTPUT_PREVIEW, outputPreview); + if (errorMessage != null) result.put(AutomationConstants.ERROR_MESSAGE, errorMessage); + return result; + } + +} diff --git a/src/prerna/reactor/automation/BuildAutomationReactor.java b/src/prerna/reactor/automation/BuildAutomationReactor.java new file mode 100644 index 00000000000..a2f8797b55e --- /dev/null +++ b/src/prerna/reactor/automation/BuildAutomationReactor.java @@ -0,0 +1,266 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.AutomationGenerationUtils; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.engine.api.IModelEngine; +import prerna.engine.impl.model.RoomUtils; +import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.run.AgentRuntimeManager; +import prerna.reactor.agent.run.RunAgentRequest; +import prerna.reactor.agent.run.RunAgentResult; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Agentic automation builder. Uses the platform RunAgent harness to generate or + * modify an automation graph JSON document from a plain-English description. + * + *

All user-accessible engines are registered as MCP tools on the Room so the + * model can query database schema, inspect engine capabilities, etc. before + * producing the final document. The model's last response must be the raw + * automation JSON document. + * + *

Pixel: {@code BuildAutomation(project=["appId"], description=["base64desc"], engine=["modelId"], currentDoc=["base64json"])} + * + *

Returns the generated document JSON as a string. Does NOT persist - caller saves via SaveAutomation. + */ +public class BuildAutomationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(BuildAutomationReactor.class); + + private static final int DESCRIPTION_MAX_CHARS = 2000; + private static final int MAX_TURNS = 10; + private static final long BUILD_TIMEOUT_MS = 180_000L; + private static final String HARNESS_TYPE = "semoss"; + + private static final String CURRENT_DOC_KEY = "currentDoc"; + + private static final String SYSTEM_PROMPT = + "You are a workflow automation builder. Your job is to create (or modify) an automation graph JSON document " + + "from a plain-English description.\n\n" + + "Use the available tools to gather context before building (for example, query a database engine to " + + "understand its schema before writing SQL). When you are done, your FINAL response must be ONLY the raw " + + "automation JSON document - no prose, no code fences, no explanation. Just: {\"version\":1,...}\n\n" + + "## Document format\n\n" + + "The document must be:\n" + + "{\"version\":1,\"description\":\"\",\"graph\":{\"nodes\":[...],\"edges\":[]}}\n\n" + + "Each node: {\"id\":\"-\",\"type\":\"\",\"label\":\"\"," + + "\"position\":{\"x\":0,\"y\":0},\"outputVar\":\"\",\"config\":{...}}\n\n" + + "Node types and config templates:\n" + + " trigger {\"mode\":\"manual\"} outputVar: trigger_out\n" + + " database-engine {\"engineId\":\"\",\"operation\":\"query\",\"expression\":\"\",\"nlPrompt\":\"\",\"limit\":50,\"commit\":false} outputVar: db_out\n" + + " model-engine {\"engineId\":\"\",\"operation\":\"llm\",\"command\":\"\",\"context\":\"\",\"paramValues\":\"\",\"values\":\"\",\"image\":\"\",\"prompt\":\"\",\"entities\":\"\"} outputVar: model_out\n" + + " vector-engine {\"engineId\":\"\",\"operation\":\"search\",\"command\":\"\",\"limit\":5,\"filters\":\"\",\"metaFilters\":\"\",\"filePath\":\"\",\"source\":\"\",\"space\":\"\",\"filePaths\":\"\",\"paramValues\":\"\",\"fileNames\":\"\"} outputVar: vector_out\n" + + " storage-engine {\"engineId\":\"\",\"operation\":\"list\",\"storagePath\":\"/\",\"filePath\":\"\",\"metadata\":\"\"} outputVar: storage_out\n" + + " function-engine {\"engineId\":\"\",\"operation\":\"execute\",\"params\":\"\"} outputVar: fn_out\n" + + " app {\"pixel\":\"\",\"appId\":\"\"} outputVar: pixel_out\n" + + " wait {\"seconds\":\"5\"} outputVar: wait_out\n\n" + + "## Building rules\n" + + "1. First node MUST be trigger: id=\"trigger-1\", type=\"trigger\".\n" + + "2. Use only node types from the list above.\n" + + "3. Set engineId from the available engines listed in the user message. Use \"\" if no match.\n" + + "4. outputVar must be unique across all nodes.\n" + + "5. Label: action-oriented verb phrase (e.g. \"Search claims database\", \"Draft summary email\").\n" + + "6. Variable substitution: reference upstream outputVars with ${varName}. In SQL wrap in single quotes: WHERE col = '${db_out}'.\n" + + "7. NEVER use SQL parameterized syntax ($1, ?, :param) - unsupported.\n" + + "8. model-engine \"command\": plain instruction only. Set \"context\" to the upstream var (e.g. \"${db_out}\") - the model receives real data at runtime.\n" + + "9. For database-engine nodes, call the get_schema tool (passing the engineId as database_id) to look up the exact table and column names before writing SQL. Never guess column names.\n" + + "10. For app nodes, use the reactor/pixel information in the user message to write valid pixel expressions.\n" + + "11. Edit mode: when an existing document is provided, preserve nodes not affected by the user's request.\n" + + "12. Format SQL with newlines and indentation: keywords (SELECT, FROM, WHERE, JOIN, ORDER BY, LIMIT, etc.) each on their own line, with clause contents indented.\n" + + "13. For every database-engine node, always include \"nlPrompt\" - a one-sentence plain-English description of what data the query fetches.\n"; + + public BuildAutomationReactor() { + this.keysToGet = new String[] { + ReactorKeysEnum.PROJECT.getKey(), + AutomationConstants.DOC_DESCRIPTION, + ReactorKeysEnum.ENGINE.getKey(), + CURRENT_DOC_KEY + }; + this.keyRequired = new int[] { 1, 1, 0, 0 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } + + String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); + String description = this.keyValue.get(AutomationConstants.DOC_DESCRIPTION); + String engineId = this.keyValue.get(ReactorKeysEnum.ENGINE.getKey()); + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access."); + } + + if (description == null || description.trim().isEmpty()) { + throw new IllegalArgumentException("A description of what the automation should do is required."); + } + try { + description = new String( + Base64.getDecoder().decode(description.trim()), + StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + // Not base64-encoded - use as-is + } + if (description.length() > DESCRIPTION_MAX_CHARS) { + description = description.substring(0, DESCRIPTION_MAX_CHARS); + } + + if (engineId == null || engineId.trim().isEmpty()) { + engineId = AutomationGenerationUtils.findFirstModelEngine(user); + } + if (engineId == null || engineId.trim().isEmpty()) { + throw new IllegalArgumentException( + "No AI model engine is available. Add a model engine connection to build an automation."); + } + if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { + throw new IllegalArgumentException( + "Model engine " + engineId + " does not exist or user does not have access."); + } + IModelEngine modelEngine = Utility.getModel(engineId); + if (modelEngine == null) { + throw new IllegalArgumentException( + "Model engine " + engineId + " could not be loaded. It may no longer exist."); + } + + String currentDocRaw = this.keyValue.get(CURRENT_DOC_KEY); + String currentDoc = null; + if (currentDocRaw != null && !currentDocRaw.trim().isEmpty()) { + try { + currentDoc = new String( + Base64.getDecoder().decode(currentDocRaw.trim()), + StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + currentDoc = currentDocRaw; + } + if (currentDoc != null && currentDoc.length() > 50_000) { + currentDoc = currentDoc.substring(0, 50_000); + } + } + + // Wrap user content to prevent prompt injection + String safeDescription = "```user-request\n" + description.trim() + "\n```"; + + StringBuilder initialMsg = new StringBuilder(); + initialMsg.append(AutomationGenerationUtils.buildAvailableEnginesSection(user)).append("\n"); + if (currentDoc != null) { + classLogger.info("BuildAutomation edit mode: project={}, docLength={}", projectId, currentDoc.length()); + // Wrap currentDoc to prevent prompt injection + String safeCurrentDoc = "```automation-json\n" + currentDoc + "\n```"; + initialMsg.append("## Existing automation to modify\n").append(safeCurrentDoc).append("\n\n"); + initialMsg.append("## User modification request\n").append(safeDescription); + } else { + initialMsg.append("## User request\n").append(safeDescription); + } + + // Fresh room per build request - isolated tool-call context, no history bleed + String pidClean = projectId.replace("-", ""); + String roomId = "automationbuild" + + pidClean.substring(0, Math.min(8, pidClean.length())) + + Long.toString(System.currentTimeMillis(), 36); + + Map options = AutomationGenerationUtils.buildEngineMcpOptions(user, SYSTEM_PROMPT); + + RoomUtils.createRoomIfNotExists(roomId, this.insight, modelEngine, initialMsg.toString(), + null, options, null, projectId, null); + + RunAgentRequest request = new RunAgentRequest( + roomId, initialMsg.toString(), engineId, HARNESS_TYPE, null, + MAX_TURNS, 0, null, null, null, null, this.insight); + + classLogger.info("BuildAutomation starting RunAgent: project={} roomId={}", projectId, roomId); + + RunAgentResult handle = AgentRuntimeManager.get().run(request); + Map result; + try { + result = AgentRuntimeManager.get().waitForRun(handle.getRunId(), this.insight, BUILD_TIMEOUT_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException("Build interrupted.", e); + } + + String status = (String) result.get("status"); + if ("FAILED".equals(status)) { + String errMsg = (String) result.get("errorMessage"); + classLogger.error("BuildAutomation run failed: project={} error={}", projectId, errMsg); + throw new RuntimeException("AI generation failed: " + (errMsg != null ? errMsg : "unknown error")); + } + + String finalText = (String) result.get("finalText"); + if (finalText == null || finalText.isBlank()) { + throw new IllegalStateException( + "The AI model did not return a response. Try again or start with a blank automation."); + } + + String docJson = AutomationGenerationUtils.stripCodeFences(finalText.trim()); + AutomationGenerationUtils.validateGeneratedDoc(docJson); + + classLogger.info("BuildAutomation finished: project={}", projectId); + return new NounMetadata(docJson, PixelDataType.CONST_STRING, PixelOperationType.OPERATION); + } + + @Override + public String getReactorDescription() { + return "Agentic automation builder using the platform RunAgent harness. Generates or edits an automation " + + "graph document from a plain-English description. All user engines are registered as MCP tools " + + "so the model can query schema and capabilities before building. Does NOT persist - " + + "caller saves via SaveAutomation."; + } + + @Override + protected String getDescriptionForKey(String key) { + return switch (key) { + case "project" -> "Project ID that will own this automation."; + case "description" -> "Base64-encoded plain-English description of what the automation should do."; + case "engine" -> "Optional model engine ID to use. Defaults to the first available MODEL engine."; + case CURRENT_DOC_KEY -> "Optional base64-encoded JSON of an existing automation document. When provided, the model modifies the existing document rather than generating from scratch."; + default -> super.getDescriptionForKey(key); + }; + } +} diff --git a/src/prerna/reactor/automation/CancelAutomationRunReactor.java b/src/prerna/reactor/automation/CancelAutomationRunReactor.java new file mode 100644 index 00000000000..49ca0c31309 --- /dev/null +++ b/src/prerna/reactor/automation/CancelAutomationRunReactor.java @@ -0,0 +1,138 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; + +/** + * Requests cancellation of a running automation. Takes effect between nodes (cannot interrupt + * mid-pixel), or mid-wait for nodes that check the flag during blocking operations. + * + *

Pixel: {@code CancelAutomationRun(project=["appId"], runId=["running-run-id"])} + * + *

Sets a cluster-safe cancellation flag ({@code AUTOMATION_RUNS.CANCEL_REQUESTED}, via + * {@link AutomationDatabaseUtility#setCancelRequested(String)}) that the executing pod's + * between-node check polls regardless of which pod that is - this is the source of truth. Also + * attempts an in-memory same-pod signal ({@link AutomationRunEngine#requestCancellation(String)}) + * as a fast path when the run happens to be executing on the pod that received this request. The + * run's {@code STATUS} is transitioned to CANCELLED by whichever pod is actually executing it, + * not by this reactor - a truly orphaned run (no pod executing it) is instead caught by the + * periodic stale-heartbeat sweep. + */ +public class CancelAutomationRunReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(CancelAutomationRunReactor.class); + + // Not standardized in ReactorKeysEnum - matches the local-key convention used by + // prerna.reactor.agent (e.g. StopAgentRunReactor.RUN_ID_KEY). + private static final String RUN_ID_KEY = "runId"; + + public CancelAutomationRunReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), RUN_ID_KEY }; + this.keyRequired = new int[] { 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + String runId = this.keyValue.get(this.keysToGet[1]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + if (runId == null || runId.isEmpty()) { + throw new IllegalArgumentException("Must provide the run id to cancel"); + } + + // Auth check + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have edit access"); + } + + // Validate the run exists, belongs to this project, and is running. Scoping by + // PROJECT_ID prevents a user with edit access to their own project from cancelling + // a run that belongs to a project they were never granted access to. + Map runDetail = AutomationDatabaseUtility.getRunDetail(runId); + if (runDetail == null || !projectId.equals(runDetail.get(AutomationConstants.PROJECT_ID))) { + throw new IllegalArgumentException("Run not found: " + runId); + } + + String status = (String) runDetail.get(AutomationConstants.STATUS); + if (!AutomationConstants.STATUS_RUNNING.equals(status)) { + throw new IllegalArgumentException( + "Can only cancel RUNNING automations. Current status: " + status); + } + + // Signal cancellation. The cluster-safe DB flag is always set - this is what the pod + // actually executing the run (which may not be this pod) polls between nodes via + // AutomationDatabaseUtility.isCancelRequested(). The in-memory signal is a same-pod fast + // path only. Unlike the prior implementation, we no longer force the run's STATUS to + // CANCELLED when the in-memory signal isn't found on this pod - the run may genuinely + // still be executing on a different pod in a cluster, and overwriting its status here + // would be a lie. The executing pod transitions STATUS to CANCELLED itself once it + // observes the flag; a truly orphaned run (crashed, nobody polling) is caught by the + // periodic stale-heartbeat sweep (AutomationDatabaseUtility.markStaleRunsInterrupted). + boolean signalledLocally = AutomationRunEngine.requestCancellation(runId); + AutomationDatabaseUtility.setCancelRequested(runId); + + classLogger.info("Cancel requested for automation run {}: signalledLocally={}", runId, signalledLocally); + + Map result = new HashMap<>(); + result.put(AutomationConstants.RUN_ID, runId); + result.put(AutomationConstants.RESULT_CANCEL_REQUESTED, true); + result.put(AutomationConstants.RESULT_SIGNALLED_LOCALLY, signalledLocally); + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + @Override + public String getReactorDescription() { + return "Requests cancellation of a running automation run for the given project."; + } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + // Cancelling a run is a mutating, side-effecting action - requires explicit confirmation. + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + return meta; + } +} diff --git a/src/prerna/reactor/automation/CreateAutomationReactor.java b/src/prerna/reactor/automation/CreateAutomationReactor.java new file mode 100644 index 00000000000..36c5fd3fc62 --- /dev/null +++ b/src/prerna/reactor/automation/CreateAutomationReactor.java @@ -0,0 +1,181 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.PixelExecutionUtils; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.project.api.IProject; +import prerna.project.impl.ProjectHelper; +import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; + +/** + * Creates a new automation project and returns its ID so the LLM can immediately chain to + * {@code QuickEditAutomation} to interactively build it. + * + *

The project name must start with a letter and contain only letters, numbers, and spaces + * (enforced by {@code CreateProject}). + * + *

Typical LLM flow: + *

    + *
  1. LLM calls {@code CreateAutomation(projectName=["My Automation"])} — auto, no UI
    2. + *
  2. LLM immediately chains {@code QuickEditAutomation(project=[""], editDescription=["..."])}
    3. + *
  3. User sees the editor open with the AI-generated draft
    4. + *
+ * + *

Pixel: {@code CreateAutomation(projectName=["My Claims Intake"])} + */ +public class CreateAutomationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(CreateAutomationReactor.class); + + private static final String PROJECT_NAME_KEY = "projectName"; + + private static final String RESULT_SUCCESS = "success"; + private static final String RESULT_PROJECT_ID = "projectId"; + private static final String RESULT_PROJECT_NAME = "projectName"; + private static final String RESULT_MESSAGE = "message"; + + public CreateAutomationReactor() { + this.keysToGet = new String[] { PROJECT_NAME_KEY }; + this.keyRequired = new int[] { 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectName = this.keyValue.get(PROJECT_NAME_KEY); + + if (projectName == null || projectName.trim().isEmpty()) { + throw new IllegalArgumentException("Must provide a projectName"); + } + projectName = projectName.trim(); + + classLogger.info("CreateAutomationReactor: creating project '{}'", projectName); + + // Validate before injecting into the pixel string — only letters, numbers, and spaces; must start + // with a letter. CreateProject enforces this too, but we reject here to prevent pixel injection. + if (!projectName.matches("^[a-zA-Z][a-zA-Z0-9 ]*$")) { + throw new IllegalArgumentException( + "Project name must start with a letter and contain only letters, numbers, and spaces. Got: " + projectName); + } + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You must be signed in to create an automation."); + } + + IProject project = ProjectHelper.generateNewProject(projectName, IProject.PROJECT_TYPE.AUTOMATION, + false, null, null, user, classLogger); + String projectId = project.getProjectId(); + String definition = buildStarterDefinition(); + String encodedDefinition = Base64.getEncoder().encodeToString(definition.getBytes(StandardCharsets.UTF_8)); + String encodedConfig = Base64.getEncoder().encodeToString( + AutomationConstants.EMPTY_JSON_ARRAY.getBytes(StandardCharsets.UTF_8)); + + try { + PixelExecutionUtils.runAndCollect(this.insight, String.format( + "SaveAutomation(project=[\"%s\"], json=[\"%s\"]);", projectId, encodedDefinition)); + PixelExecutionUtils.runAndCollect(this.insight, String.format( + "SaveAutomationConfig(project=[\"%s\"], config=[\"%s\"]);", projectId, encodedConfig)); + MCPUtility.addMCPTag(project); + } catch (PixelExecutionUtils.AutomationPixelException e) { + classLogger.error("Failed to scaffold automation project '{}'", projectName, e); + throw new IllegalStateException( + "Automation project was created but its starter assets could not be scaffolded: " + e.getMessage(), e); + } + + classLogger.info("CreateAutomationReactor: created project '{}' with id {}", projectName, projectId); + + Map result = new LinkedHashMap<>(); + result.put(RESULT_SUCCESS, true); + result.put(RESULT_PROJECT_ID, projectId); + result.put("project_id", projectId); + result.put(RESULT_PROJECT_NAME, projectName); + result.put(RESULT_MESSAGE, + "Created automation project \"" + projectName + "\" (id: " + projectId + "). " + + "Call QuickEditAutomation(project=[\"" + projectId + + "\"], editDescription=[\"\"]) to build the automation."); + + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + private static String buildStarterDefinition() { + Map trigger = new LinkedHashMap<>(); + trigger.put(AutomationConstants.NODE_FIELD_ID, "trigger"); + trigger.put(AutomationConstants.NODE_FIELD_TYPE, AutomationConstants.NODE_TRIGGER); + trigger.put(AutomationConstants.NODE_FIELD_LABEL, "Start"); + trigger.put(AutomationConstants.NODE_FIELD_CONFIG, Map.of()); + + Map graph = new LinkedHashMap<>(); + graph.put(AutomationConstants.DOC_NODES, java.util.List.of(trigger)); + graph.put(AutomationConstants.DOC_EDGES, java.util.List.of()); + + Map definition = new LinkedHashMap<>(); + definition.put(AutomationConstants.DOC_VERSION, AutomationConstants.DOC_CURRENT_VERSION); + definition.put(AutomationConstants.DOC_DESCRIPTION, ""); + definition.put(AutomationConstants.DOC_GRAPH, graph); + return prerna.reactor.automation.utils.AutomationExecutionUtils.GSON.toJson(definition); + } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + return meta; + } + + @Override + public String getReactorDescription() { + return "Creates a new blank automation project and returns its ID. " + + "Immediately chain QuickEditAutomation with the returned project ID to build the automation. " + + "Project names must start with a letter and contain only letters, numbers, and spaces."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (PROJECT_NAME_KEY.equals(key)) { + return "Display name for the new automation project. " + + "Must start with a letter and contain only letters, numbers, and spaces."; + } + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/reactor/automation/ExplainAutomationReactor.java b/src/prerna/reactor/automation/ExplainAutomationReactor.java new file mode 100644 index 00000000000..3f6152b6a66 --- /dev/null +++ b/src/prerna/reactor/automation/ExplainAutomationReactor.java @@ -0,0 +1,209 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.AutomationGenerationUtils; + +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.engine.api.IModelEngine; +import prerna.reactor.AbstractReactor; +import prerna.reactor.automation.utils.AutomationExecutionUtils; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Generates a plain-English explanation of what an automation does when run. + * Reads the saved automation.json, sends it to a model engine with a narration prompt, + * and returns the explanation as a string. + * + *

Suitable for sharing with non-technical stakeholders or onboarding new team members. + * + *

Uses the same model-engine resolution logic as {@link BuildAutomationReactor}: + * uses the {@code engine} param if provided, otherwise falls back to the first accessible MODEL engine. + * + *

Pixel: {@code ExplainAutomation(project=["appId"])} + */ +public class ExplainAutomationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(ExplainAutomationReactor.class); + + /** + * Narration prompt for the full explain mode (saved doc, 2-3 sentences for stakeholders). + * Used when no {@code content} parameter is provided. + */ + private static final String NARRATION_SYSTEM_PROMPT = + "You are a helpful assistant that explains software automation workflows to non-technical users. " + + "Given the following automation JSON definition, write a 2-3 sentence plain-English explanation " + + "of what this automation does when it runs. " + + "Write for a non-technical audience - do not mention JSON, nodes, config keys, or technical terms. " + + "Start your response with: 'When you run this automation, it will...'"; + + /** + * Suggest prompt for the description field (in-memory doc, 1 sentence for the description field). + * Used when a {@code content} parameter is provided (current unsaved state from the FE). + */ + private static final String SUGGEST_SYSTEM_PROMPT = + "You generate one-sentence descriptions for workflow automations. " + + "Given an automation graph JSON, write a single clear sentence (under 20 words) describing what the automation does. " + + "Be specific about the actions taken, not about node types or technical structure. " + + "Start with an active verb. No quotes, no period at the end, no explanation. " + + "Examples: 'Queries open claims and drafts a daily summary email for case managers', " + + "'Searches the knowledge base for relevant documents and generates a response using AI', " + + "'Uploads processed files to cloud storage and updates the claims database'."; + + private static final String CONTENT_KEY = "content"; + + public ExplainAutomationReactor() { + this.keysToGet = new String[] { + ReactorKeysEnum.PROJECT.getKey(), + ReactorKeysEnum.ENGINE.getKey(), + CONTENT_KEY, + }; + this.keyRequired = new int[] { 1, 0, 0 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } + + String projectId = this.keyValue.get(this.keysToGet[0]); + String engineId = this.keyValue.get(this.keysToGet[1]); + String contentEncoded = this.keyValue.get(CONTENT_KEY); + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { + throw new IllegalArgumentException( + "Project does not exist or user does not have access."); + } + + // Determine mode: suggest (in-memory content provided) vs. narrate (read saved file) + boolean suggestMode = contentEncoded != null && !contentEncoded.trim().isEmpty(); + String doc; + String systemPrompt; + if (suggestMode) { + try { + doc = new String( + java.util.Base64.getDecoder().decode(contentEncoded.trim()), + StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + doc = contentEncoded; // not base64-encoded, use as-is + } + if (doc.length() > 50_000) { + doc = doc.substring(0, 50_000); + } + systemPrompt = SUGGEST_SYSTEM_PROMPT; + classLogger.info("ExplainAutomationReactor: suggest mode (in-memory content), project={}", projectId); + } else { + classLogger.info("ExplainAutomationReactor: narrate mode (saved file), project={}", projectId); + doc = AutomationExecutionUtils.loadAutomationDocOrEmpty(projectId); + if (doc.length() > 50_000) { + doc = doc.substring(0, 50_000); + } + systemPrompt = NARRATION_SYSTEM_PROMPT; + } + + // Resolve model engine - provided ID or first accessible MODEL engine + if (engineId == null || engineId.trim().isEmpty()) { + engineId = AutomationGenerationUtils.findFirstModelEngine(user); + } + if (engineId == null || engineId.trim().isEmpty()) { + throw new IllegalArgumentException( + "No AI model engine is available. Add a model engine connection to use this feature."); + } + if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { + throw new IllegalArgumentException( + "Model engine " + engineId + " does not exist or user does not have access."); + } + + IModelEngine modelEngine = Utility.getModel(engineId); + if (modelEngine == null) { + throw new IllegalArgumentException( + "Model engine " + engineId + " could not be loaded. It may no longer exist."); + } + + classLogger.info("ExplainAutomationReactor: calling model engine {} for project {}", engineId, projectId); + + Map paramMap = new HashMap<>(); + paramMap.put("use_history", false); + + Map response; + try { + String userMessage = "Automation definition:\n" + doc; + response = modelEngine.ask(systemPrompt + "\n\n" + userMessage, + null, this.insight, paramMap).toMap(); + } catch (Exception e) { + classLogger.error("LLM call failed for ExplainAutomation on project {}", projectId, e); + throw new RuntimeException("AI explanation failed: " + e.getMessage(), e); + } + + String explanation = AutomationGenerationUtils.extractResponseText(response); + if (explanation == null || explanation.isBlank()) { + throw new IllegalStateException( + "The AI model did not return an explanation. Try again."); + } + + classLogger.info("ExplainAutomationReactor: completed for project {}", projectId); + return new NounMetadata(explanation.trim(), PixelDataType.CONST_STRING, PixelOperationType.OPERATION); + } + + @Override + public String getReactorDescription() { + return "Generates a plain-English explanation of what a saved automation does when run, " + + "suitable for sharing with non-technical stakeholders. " + + "Reads the saved automation.json and narrates it using a model engine."; + } + + @Override + protected String getDescriptionForKey(String key) { + return switch (key) { + case "project" -> "The project ID of the automation to explain."; + case "engine" -> "Optional model engine ID to use. Defaults to the first accessible MODEL engine."; + case CONTENT_KEY -> "Optional base64-encoded JSON of the current automation (from FE in-memory state). " + + "When provided, produces a one-sentence description suitable for the description field " + + "instead of reading the saved file and narrating 2-3 sentences."; + default -> super.getDescriptionForKey(key); + }; + } +} diff --git a/src/prerna/reactor/automation/GenerateNodeLabelReactor.java b/src/prerna/reactor/automation/GenerateNodeLabelReactor.java new file mode 100644 index 00000000000..3fe6d345dad --- /dev/null +++ b/src/prerna/reactor/automation/GenerateNodeLabelReactor.java @@ -0,0 +1,248 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.AutomationGenerationUtils; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.engine.api.IModelEngine; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Generates a short, action-oriented label (2–4 words) for an automation node + * based on its type and configuration. Uses the first available MODEL engine. + * + *

Pixel: {@code GenerateNodeLabel(project=["appId"], type=["model-engine"], config=["base64json"])} + */ +public class GenerateNodeLabelReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GenerateNodeLabelReactor.class); + + private static final String KEY_TYPE = "type"; + private static final String KEY_CONFIG = "config"; + + private static final String SYSTEM_PROMPT = + "You generate short, action-oriented labels for workflow automation steps. " + + "Given the step type and key configuration details, respond with ONLY the label - " + + "2 to 4 words maximum, plain English, no quotes, no punctuation at the end, no explanation. " + + "The label should describe what the step DOES, not what type of node it is. " + + "Examples: 'Search claims data', 'Draft email reply', 'Fetch open tickets', 'Summarize results', " + + "'Upload report file', 'Query veteran records', 'Pause 30 seconds'."; + + public GenerateNodeLabelReactor() { + this.keysToGet = new String[] { + ReactorKeysEnum.PROJECT.getKey(), + KEY_TYPE, + KEY_CONFIG, + }; + this.keyRequired = new int[] { 1, 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } + + String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); + String nodeType = this.keyValue.get(KEY_TYPE); + String configEncoded = this.keyValue.get(KEY_CONFIG); + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access."); + } + + // Decode base64 config JSON sent by FE + String configJson; + try { + configJson = new String(Base64.getDecoder().decode(configEncoded.trim()), java.nio.charset.StandardCharsets.UTF_8); + } catch (IllegalArgumentException e) { + configJson = configEncoded; + } + + String engineId = AutomationGenerationUtils.findFirstModelEngine(user); + if (engineId == null || engineId.isBlank()) { + throw new IllegalArgumentException( + "No AI model engine is available. Add a model engine connection to use this feature."); + } + if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { + throw new IllegalArgumentException( + "Model engine " + engineId + " does not exist or user does not have access."); + } + + IModelEngine modelEngine = Utility.getModel(engineId); + if (modelEngine == null) { + throw new IllegalArgumentException( + "Model engine " + engineId + " could not be loaded."); + } + + String userMessage = buildUserMessage(nodeType, configJson); + classLogger.info("GenerateNodeLabel: project={}, type={}", projectId, nodeType); + + Map paramMap = new HashMap<>(); + paramMap.put("use_history", false); + + Map response; + try { + response = modelEngine.ask(SYSTEM_PROMPT + "\n\n" + userMessage, + null, this.insight, paramMap).toMap(); + } catch (Exception e) { + classLogger.error("LLM call failed for GenerateNodeLabel on project {}", projectId, e); + throw new RuntimeException("Label generation failed: " + e.getMessage(), e); + } + + String label = AutomationGenerationUtils.extractResponseText(response); + if (label == null || label.isBlank()) { + throw new IllegalStateException("The AI model did not return a label. Try again."); + } + + // Strip surrounding quotes the model sometimes adds, and cap length + label = label.strip().replaceAll("^[\"']+|[\"']+$", ""); + if (label.length() > 50) { + label = label.substring(0, 50); + } + + return new NounMetadata(label, PixelDataType.CONST_STRING, PixelOperationType.OPERATION); + } + + /** + * Builds a human-readable summary of the node config for the LLM prompt. + * Skips engine IDs and other technical fields the model doesn't need to understand. + */ + @SuppressWarnings("unchecked") + private static String buildUserMessage(String nodeType, String configJson) { + StringBuilder sb = new StringBuilder(); + sb.append("Step type: ").append(humanNodeType(nodeType)).append("\n"); + + try { + Map cfg = AutomationExecutionUtils.GSON.fromJson(configJson, AutomationExecutionUtils.MAP_TYPE); + if (cfg == null) { + sb.append("Config: (none)"); + return sb.toString(); + } + + switch (nodeType) { + case "model-engine" -> { + appendField(sb, "Instruction", cfg.get("command")); + appendField(sb, "Input data", cfg.get("context")); + appendField(sb, "Operation", cfg.get("operation")); + } + case "database-engine" -> { + appendField(sb, "SQL query", cfg.get("expression")); + appendField(sb, "Operation", cfg.get("operation")); + } + case "vector-engine" -> { + appendField(sb, "Search query", cfg.get("command")); + appendField(sb, "Operation", cfg.get("operation")); + appendField(sb, "File", cfg.get("filePath")); + } + case "storage-engine" -> { + appendField(sb, "Operation", cfg.get("operation")); + appendField(sb, "Path", cfg.get("storagePath")); + appendField(sb, "File", cfg.get("filePath")); + } + case "function-engine" -> { + appendField(sb, "Parameters", cfg.get("params")); + appendField(sb, "Operation", cfg.get("operation")); + } + case "app" -> { + appendField(sb, "Pixel expression", cfg.get("pixel")); + } + case "wait" -> { + appendField(sb, "Seconds", cfg.get("seconds")); + } + default -> { + // No config details for trigger or unknown types + } + } + } catch (Exception e) { + sb.append("Config: (unparseable)"); + } + + return sb.toString(); + } + + private static void appendField(StringBuilder sb, String fieldName, Object value) { + if (value == null) return; + String s = value.toString().trim(); + if (s.isEmpty() || s.equals("PENDING_SQL_GENERATION") || s.equals("PENDING_PIXEL_EXPRESSION")) return; + // Truncate very long values so they don't dominate the prompt + if (s.length() > 200) s = s.substring(0, 200) + "..."; + sb.append(fieldName).append(": ").append(s).append("\n"); + } + + private static String humanNodeType(String type) { + return switch (type) { + case "model-engine" -> "Ask AI (model engine)"; + case "database-engine" -> "Query database"; + case "vector-engine" -> "Search documents (vector engine)"; + case "storage-engine" -> "File storage"; + case "function-engine" -> "Custom function"; + case "app" -> "Run pixel / app reactor"; + case "wait" -> "Pause / wait"; + case "trigger" -> "Trigger"; + default -> type; + }; + } + + @Override + public String getReactorDescription() { + return "Generates a short 2–4 word action-oriented label for an automation node based on its type and config. " + + "Uses the first available MODEL engine. Returns the suggested label as a string."; + } + + @Override + protected String getDescriptionForKey(String key) { + return switch (key) { + case "project" -> "The project ID the automation belongs to."; + case "type" -> "The node type (e.g. model-engine, database-engine, vector-engine)."; + case "config" -> "Base64-encoded JSON of the node's current config object."; + default -> super.getDescriptionForKey(key); + }; + } +} diff --git a/src/prerna/reactor/automation/GenerateRunSummaryReactor.java b/src/prerna/reactor/automation/GenerateRunSummaryReactor.java new file mode 100644 index 00000000000..00bd7f36412 --- /dev/null +++ b/src/prerna/reactor/automation/GenerateRunSummaryReactor.java @@ -0,0 +1,167 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.AutomationGenerationUtils; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.engine.api.IModelEngine; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Generates a plain-English summary of a completed automation run using an LLM. + * Reads per-step output previews from the database and asks the model to describe + * what happened in 2-3 sentences suitable for a non-technical user. + * + *

Pixel: {@code GenerateRunSummary(project=["appId"], runId=["runId"])} + */ +public class GenerateRunSummaryReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GenerateRunSummaryReactor.class); + + private static final String KEY_RUN_ID = "runId"; + + private static final String SYSTEM_PROMPT = + "You summarize completed automation workflow runs for non-technical users. " + + "Given the run status and a list of steps with their outputs, write a 2-3 sentence " + + "plain-English summary of what happened: what the automation did, what it found or produced, " + + "and (if applicable) what went wrong. Be specific about data where available. " + + "Do not use technical jargon, node IDs, or implementation details. " + + "Do not start with 'The automation'. Respond with only the summary - no preamble, no headers."; + + public GenerateRunSummaryReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), KEY_RUN_ID }; + this.keyRequired = new int[] { 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } + + String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); + String runId = this.keyValue.get(KEY_RUN_ID); + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access."); + } + + Map runDetail = AutomationDatabaseUtility.getRunDetail(runId); + if (runDetail == null) { + throw new IllegalArgumentException("Run not found: " + runId); + } + Object runProjectId = runDetail.get(AutomationConstants.PROJECT_ID); + if (!projectId.equals(runProjectId)) { + throw new IllegalArgumentException("Run not found: " + runId); + } + + // Guard against summarizing an in-progress run - the result would be incomplete. + Object runStatus = runDetail.get(AutomationConstants.STATUS); + if (AutomationConstants.STATUS_RUNNING.equals(runStatus)) { + throw new IllegalArgumentException("Run " + runId + " is still in progress. Wait for it to complete."); + } + + List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(runId); + if (nodeOutputs.isEmpty()) { + classLogger.warn("GenerateRunSummary: no node outputs found for run {}", runId); + } + + String engineId = AutomationGenerationUtils.findFirstModelEngine(user); + if (engineId == null || engineId.isBlank()) { + throw new IllegalArgumentException( + "No AI model engine is available. Add a model engine connection to use this feature."); + } + if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { + throw new IllegalArgumentException( + "Model engine " + engineId + " does not exist or user does not have access."); + } + + IModelEngine modelEngine = Utility.getModel(engineId); + if (modelEngine == null) { + throw new IllegalArgumentException("Model engine " + engineId + " could not be loaded."); + } + + String userMessage = AutomationExecutionUtils.buildRunSummaryPrompt(runDetail, nodeOutputs); + + Map paramMap = new HashMap<>(); + paramMap.put("use_history", false); + + Map response; + try { + response = modelEngine.ask(SYSTEM_PROMPT + "\n\n" + userMessage, + null, this.insight, paramMap).toMap(); + } catch (Exception e) { + classLogger.error("LLM call failed for GenerateRunSummary on run {}", runId, e); + throw new RuntimeException("Summary generation failed: " + e.getMessage(), e); + } + + String summary = AutomationGenerationUtils.extractResponseText(response); + if (summary == null || summary.isBlank()) { + throw new IllegalStateException("The AI model did not return a summary. Try again."); + } + + classLogger.info("GenerateRunSummary completed: project={}, runId={}", projectId, runId); + return new NounMetadata(summary.strip(), PixelDataType.CONST_STRING, PixelOperationType.OPERATION); + } + + @Override + public String getReactorDescription() { + return "Generates a plain-English 2-3 sentence summary of a completed automation run " + + "using an LLM. Reads per-step results from the database. Returns the summary string."; + } + + @Override + protected String getDescriptionForKey(String key) { + return switch (key) { + case "project" -> "The project ID the automation belongs to."; + case "runId" -> "The run ID to summarize."; + default -> super.getDescriptionForKey(key); + }; + } +} diff --git a/src/prerna/reactor/automation/GetActiveAutomationRunReactor.java b/src/prerna/reactor/automation/GetActiveAutomationRunReactor.java new file mode 100644 index 00000000000..d977d5bb0b4 --- /dev/null +++ b/src/prerna/reactor/automation/GetActiveAutomationRunReactor.java @@ -0,0 +1,102 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; + +/** + * Returns the currently active run ID for a project by reading the + * {@code AUTOMATION_ACTIVE_RUN} lock table directly. This table is populated by + * {@link AutomationDatabaseUtility#claimActiveRun} before + * {@code AUTOMATION_RUNS} is written, so the FE can discover the run ID while + * {@link TriggerAutomationReactor} is still executing synchronously on a virtual + * thread. + * + *

Returns {@code { RUN_ID, PROJECT_ID }} when a run is active, or an empty + * map when no run is in progress. + * + *

Pixel: {@code GetActiveAutomationRun(project=["appId"])} + */ +public class GetActiveAutomationRunReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GetActiveAutomationRunReactor.class); + + public GetActiveAutomationRunReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey() }; + this.keyRequired = new int[] { 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = getProjectId(); + + String runId = AutomationDatabaseUtility.getClaimedActiveRun(projectId); + + Map result = new HashMap<>(); + if (runId != null) { + result.put(AutomationConstants.RUN_ID, runId); + result.put(AutomationConstants.PROJECT_ID, projectId); + } + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + private String getProjectId() { + String projectId = this.keyValue.get(this.keysToGet[0]); + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + return projectId; + } + + @Override + public String getReactorDescription() { + return "Returns the active run ID from the AUTOMATION_ACTIVE_RUN table; empty map when no run is in progress."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) return "The project (app) ID or alias to check for an active run."; + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/reactor/automation/GetAutomationConfigReactor.java b/src/prerna/reactor/automation/GetAutomationConfigReactor.java new file mode 100644 index 00000000000..5f1f4d55ee6 --- /dev/null +++ b/src/prerna/reactor/automation/GetAutomationConfigReactor.java @@ -0,0 +1,124 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.reflect.TypeToken; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.util.Utility; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.AssetUtility; + +/** + * Returns the automation environment config ({@code automation_config.json}) for a project. + * Sensitive values are masked in the response - only the key and a placeholder are returned. + * + *

There are three "get" reactors - each reads something different: + *

    + *
  • {@link GetAutomationReactor GetAutomation} - reads {@code automation.json}: the pipeline graph
    • + *
  • {@code GetAutomationConfig} (this reactor) - reads {@code automation_config.json}: key/value env vars and secrets
    • + *
  • {@link GetAutomationRunReactor GetAutomationRun} - reads live run state from the DB
    • + *
+ * + *

Pixel: {@code GetAutomationConfig(project=["appId"])} + */ +public class GetAutomationConfigReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GetAutomationConfigReactor.class); + + public GetAutomationConfigReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey() }; + this.keyRequired = new int[] { 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File configFile = Paths.get(portalsFolder, AutomationConstants.AUTOMATION_CONFIG_FILE_NAME).toFile(); + String normalizedPath = Utility.normalizePath(configFile.getAbsolutePath()); + if (!normalizedPath.startsWith(portalsFolder)) { + throw new IllegalArgumentException("Invalid file path"); + } + + if (!configFile.exists() || !configFile.isFile()) { + return new NounMetadata(new ArrayList<>(), PixelDataType.VECTOR, PixelOperationType.OPERATION); + } + + try { + String json = Files.readString(configFile.toPath(), StandardCharsets.UTF_8); + List> entries = AutomationExecutionUtils.GSON.fromJson(json, new TypeToken>>() {}.getType()); + if (entries != null) { + for (Map entry : entries) { + Object sensitive = entry.get(AutomationConstants.CONFIG_ENTRY_SENSITIVE); + if (Boolean.TRUE.equals(sensitive)) { + entry.put(AutomationConstants.CONFIG_ENTRY_VALUE, AutomationConstants.SENSITIVE_MASK); + } + } + } + return new NounMetadata(entries != null ? entries : new ArrayList<>(), PixelDataType.VECTOR, PixelOperationType.OPERATION); + } catch (IOException e) { + classLogger.error("Error reading automation_config.json for project {}", projectId, e); + return new NounMetadata(new ArrayList<>(), PixelDataType.VECTOR, PixelOperationType.OPERATION); + } + } + + @Override + public String getReactorDescription() { + return "Returns the automation environment config (automation_config.json) for a project; sensitive values are masked."; + } +} diff --git a/src/prerna/reactor/automation/GetAutomationReactor.java b/src/prerna/reactor/automation/GetAutomationReactor.java new file mode 100644 index 00000000000..6b383912ef5 --- /dev/null +++ b/src/prerna/reactor/automation/GetAutomationReactor.java @@ -0,0 +1,139 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.project.api.IProject; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.AssetUtility; +import prerna.util.Utility; + +/** + * Returns the automation definition (the saved {@code automation.json} graph) for a project. + * Returns an empty graph document when no automation has been saved yet. + * + *

There are three "get" reactors - each reads something different: + *

    + *
  • {@code GetAutomation} (this reactor) - reads {@code automation.json}: the pipeline graph + * (nodes, edges, output transforms). Static config written by {@link SaveAutomationReactor}.
    • + *
  • {@link GetAutomationConfigReactor GetAutomationConfig} - reads {@code automation_config.json}: + * key/value env vars and secrets; sensitive values are masked in the response.
    • + *
  • {@link GetAutomationRunReactor GetAutomationRun} - reads live run state from the DB + * (AUTOMATION_RUNS + AUTOMATION_NODE_OUTPUTS); used by the FE to poll execution progress.
    • + *
+ * + *

Pixel: {@code GetAutomation(project=["appId"])} + */ +public class GetAutomationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GetAutomationReactor.class); + + public GetAutomationReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey() }; + this.keyRequired = new int[] { 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + + IProject project = Utility.getProject(projectId); + if (project != null && project.requirePublish(true)) { + classLogger.info("Pulled project {} from cluster", projectId); + } + + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File automationFile = Paths.get(portalsFolder, AutomationConstants.AUTOMATION_FILE_NAME).toFile(); + String normalizedPath = Utility.normalizePath(automationFile.getAbsolutePath()); + if (!normalizedPath.startsWith(portalsFolder)) { + throw new IllegalArgumentException("Invalid file path"); + } + + if (!automationFile.exists() || !automationFile.isFile()) { + // return empty graph document for brand-new automations + Map empty = new HashMap<>(); + empty.put(AutomationConstants.DOC_VERSION, AutomationConstants.DOC_CURRENT_VERSION); + Map graph = new HashMap<>(); + graph.put(AutomationConstants.DOC_NODES, new ArrayList<>()); + graph.put(AutomationConstants.DOC_EDGES, new ArrayList<>()); + empty.put(AutomationConstants.DOC_GRAPH, graph); + return new NounMetadata(empty, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + try { + String json = Files.readString(automationFile.toPath(), StandardCharsets.UTF_8); + Map doc = AutomationExecutionUtils.GSON.fromJson(json, + AutomationExecutionUtils.MAP_TYPE); + return new NounMetadata(doc, PixelDataType.MAP, PixelOperationType.OPERATION); + } catch (IOException e) { + classLogger.error("Error reading automation.json for project {}", projectId, e); + throw new IllegalArgumentException("Unable to read automation: " + e.getMessage()); + } + } + + @Override + public String getReactorDescription() { + return "Returns the automation pipeline definition (automation.json) for a project; returns an empty graph when none has been saved."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) { + return "The project ID of the automation to retrieve."; + } + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/reactor/automation/GetAutomationRunReactor.java b/src/prerna/reactor/automation/GetAutomationRunReactor.java new file mode 100644 index 00000000000..939c1f77c4e --- /dev/null +++ b/src/prerna/reactor/automation/GetAutomationRunReactor.java @@ -0,0 +1,104 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; + +/** + * Returns detail for a single automation run including per-node results. + * + *

Pixel: {@code GetAutomationRun(app=["appId"], runId=["uuid"])} + * + *

Reads from AUTOMATION_RUNS and AUTOMATION_NODE_OUTPUTS in the scheduler DB. + */ +public class GetAutomationRunReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GetAutomationRunReactor.class); + + // Not standardized in ReactorKeysEnum — matches the local-key convention used by + // prerna.reactor.agent (e.g. GetAgentRunReactor.RUN_ID_KEY). + private static final String RUN_ID_KEY = "runId"; + + public GetAutomationRunReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), RUN_ID_KEY }; + this.keyRequired = new int[] { 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + String runId = this.keyValue.get(this.keysToGet[1]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + if (runId == null || runId.isEmpty()) { + throw new IllegalArgumentException("Must provide a run id"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + + Map runDetail = AutomationDatabaseUtility.getRunDetail(runId); + // Scope by PROJECT_ID so a user with view access to one project cannot read another + // project's run detail/node outputs by guessing or reusing a runId. + if (runDetail == null || !projectId.equals(runDetail.get(AutomationConstants.PROJECT_ID))) { + Map notFound = new HashMap<>(); + notFound.put(AutomationConstants.RUN_ID, runId); + notFound.put(AutomationConstants.RESULT_NODE_RESULTS, new ArrayList<>()); + return new NounMetadata(notFound, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(runId); + List> nodeResults = AutomationDatabaseUtility.buildNodeResults(nodeOutputs); + + runDetail.put(AutomationConstants.RESULT_NODE_RESULTS, nodeResults); + return new NounMetadata(runDetail, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + @Override + public String getReactorDescription() { + return "Returns detail for a single automation run, including per-node results."; + } +} diff --git a/src/prerna/reactor/automation/GetAutomationSchemaReactor.java b/src/prerna/reactor/automation/GetAutomationSchemaReactor.java new file mode 100644 index 00000000000..f4351ef8e26 --- /dev/null +++ b/src/prerna/reactor/automation/GetAutomationSchemaReactor.java @@ -0,0 +1,188 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.engine.api.IDatabaseEngine; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Returns the physical table and column names for every database-engine node + * in the automation that exposes its SQL expression as a playground-fillable + * field. Used by the LLM to discover the schema before writing SQL queries for + * {@code TriggerAutomation}. + * + *

Pixel: {@code GetAutomationSchema(project=["appId"])} + */ +public class GetAutomationSchemaReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GetAutomationSchemaReactor.class); + + public GetAutomationSchemaReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey() }; + this.keyRequired = new int[] { 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } + String projectId = this.keyValue.get(this.keysToGet[0]); + if (projectId == null || projectId.isBlank()) { + throw new IllegalArgumentException("Must provide a project id"); + } + projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + + Map doc = AutomationExecutionUtils.loadAutomationDoc(projectId); + @SuppressWarnings("unchecked") + Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); + @SuppressWarnings("unchecked") + List> nodes = graph != null ? (List>) graph.get(AutomationConstants.DOC_NODES) : null; + + List> result = new ArrayList<>(); + if (nodes != null) { + for (Map node : nodes) { + if (!AutomationConstants.NODE_DATABASE_ENGINE.equals(node.get(AutomationConstants.NODE_FIELD_TYPE))) { + continue; + } + @SuppressWarnings("unchecked") + List fillable = (List) node.get("playgroundFillable"); + if (fillable == null || !fillable.contains(AutomationConstants.CONFIG_EXPRESSION)) { + continue; + } + @SuppressWarnings("unchecked") + Map config = (Map) node.get(AutomationConstants.NODE_FIELD_CONFIG); + if (config == null) continue; + + String nodeLabel = (String) node.get(AutomationConstants.NODE_FIELD_LABEL); + Object engineIdObj = config.get(AutomationConstants.CONFIG_ENGINE_ID); + if (engineIdObj == null || engineIdObj.toString().isBlank()) continue; + + String engineId = engineIdObj.toString(); + if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { + throw new IllegalArgumentException( + "Database engine configured for node '" + nodeLabel + "' is not accessible."); + } + Map nodeSchema = buildNodeSchema(nodeLabel, engineId); + if (nodeSchema != null) { + result.add(nodeSchema); + } + } + } + + Map output = new HashMap<>(); + output.put("nodes", result); + if (result.isEmpty()) { + output.put("message", "No database nodes with playground-fillable SQL expressions found in this automation."); + } + return new NounMetadata(output, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + private Map buildNodeSchema(String nodeLabel, String engineId) { + IDatabaseEngine engine; + try { + engine = Utility.getDatabase(engineId); + } catch (Exception e) { + classLogger.warn("Could not load database engine {} for schema discovery", engineId, e); + return null; + } + if (engine == null) { + classLogger.warn("Database engine {} not found for node '{}'", engineId, nodeLabel); + return null; + } + + List conceptUris = engine.getPhysicalConcepts(); + if (conceptUris == null || conceptUris.isEmpty()) { + return null; + } + + List> tables = new ArrayList<>(); + for (String conceptUri : conceptUris) { + String tableName = Utility.getInstanceName(conceptUri); + if (tableName == null || tableName.isBlank()) continue; + + List columns = new ArrayList<>(); + List propUris = engine.getPropertyUris4PhysicalUri(conceptUri); + if (propUris != null) { + for (String propUri : propUris) { + String colName = Utility.getInstanceName(propUri); + if (colName != null && !colName.isBlank()) { + columns.add(colName); + } + } + } + + Map tableEntry = new HashMap<>(); + tableEntry.put("name", tableName); + tableEntry.put("columns", columns); + tables.add(tableEntry); + } + + if (tables.isEmpty()) return null; + + Map nodeSchema = new HashMap<>(); + nodeSchema.put("nodeLabel", nodeLabel != null ? nodeLabel : AutomationConstants.UNNAMED_NODE_LABEL); + nodeSchema.put("engineId", engineId); + nodeSchema.put("tables", tables); + return nodeSchema; + } + + @Override + public String getReactorDescription() { + return "Returns the physical table and column names for database-engine nodes in the automation that accept SQL input. Call this before TriggerAutomation to discover the exact table and column names to use in your SQL query."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) return "The project (app) ID or alias to retrieve the database schema for."; + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/reactor/automation/GetAutomationStructureReactor.java b/src/prerna/reactor/automation/GetAutomationStructureReactor.java new file mode 100644 index 00000000000..c83c3f8ca4d --- /dev/null +++ b/src/prerna/reactor/automation/GetAutomationStructureReactor.java @@ -0,0 +1,163 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.io.File; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.reactor.automation.utils.AutomationExecutionUtils; +import prerna.util.Utility; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.AssetUtility; + +/** + * Returns a compact summary of the automation's current structure — description and a flat + * node list (label, type, operation, engineId) — for use as LLM context before editing. + * + *

Trigger nodes are excluded; they are infrastructure, not user-authored steps. + * Returns {@code { description: "", nodes: [] }} for a blank or missing automation. + * + *

Pixel: {@code GetAutomationStructure(project=["appId"])} + */ +public class GetAutomationStructureReactor extends AbstractReactor { + + private static final String RESULT_KEY_DESCRIPTION = "description"; + private static final String RESULT_KEY_NODES = "nodes"; + + public GetAutomationStructureReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey() }; + this.keyRequired = new int[] { 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + + Map result = new LinkedHashMap<>(); + result.put(RESULT_KEY_DESCRIPTION, ""); + result.put(RESULT_KEY_NODES, new ArrayList<>()); + + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File automationFile = Paths.get(portalsFolder, AutomationConstants.AUTOMATION_FILE_NAME).toFile(); + String normalizedPath = Utility.normalizePath(automationFile.getAbsolutePath()); + if (!normalizedPath.startsWith(portalsFolder)) { + throw new IllegalArgumentException("Invalid file path"); + } + + if (!automationFile.exists() || !automationFile.isFile()) { + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + Map doc = AutomationExecutionUtils.loadAutomationDoc(projectId); + + String description = (String) doc.getOrDefault(AutomationConstants.DOC_DESCRIPTION, ""); + result.put(RESULT_KEY_DESCRIPTION, description != null ? description : ""); + + Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); + if (graph != null) { + List nodes = (List) graph.get(AutomationConstants.DOC_NODES); + if (nodes != null) { + result.put(RESULT_KEY_NODES, buildNodeSummary(nodes)); + } + } + + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + /** + * Extracts a compact summary for each non-trigger node. + * Preserves label, type, and the two most useful config fields for LLM context. + */ + @SuppressWarnings("unchecked") + private static List> buildNodeSummary(List nodes) { + List> summary = new ArrayList<>(); + for (Object raw : nodes) { + if (!(raw instanceof Map)) { + continue; + } + Map node = (Map) raw; + String type = (String) node.get(AutomationConstants.NODE_FIELD_TYPE); + + // Trigger nodes are not user-authored steps — skip them + if (AutomationConstants.NODE_TRIGGER.equals(type)) { + continue; + } + + Map entry = new LinkedHashMap<>(); + entry.put(AutomationConstants.NODE_FIELD_LABEL, + node.getOrDefault(AutomationConstants.NODE_FIELD_LABEL, AutomationConstants.UNNAMED_NODE_LABEL)); + entry.put(AutomationConstants.NODE_FIELD_TYPE, type != null ? type : ""); + + Map config = (Map) node.get(AutomationConstants.NODE_FIELD_CONFIG); + if (config != null) { + if (config.containsKey(AutomationConstants.CONFIG_OPERATION)) { + entry.put(AutomationConstants.CONFIG_OPERATION, config.get(AutomationConstants.CONFIG_OPERATION)); + } + if (config.containsKey(AutomationConstants.CONFIG_ENGINE_ID)) { + entry.put(AutomationConstants.CONFIG_ENGINE_ID, config.get(AutomationConstants.CONFIG_ENGINE_ID)); + } + } + + summary.add(entry); + } + return summary; + } + + @Override + public String getReactorDescription() { + return "Returns the automation's description and a compact node list (label, type, operation, engineId) " + + "for LLM context. Trigger nodes are excluded. Returns an empty node list for a blank automation."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) { + return "The project ID of the automation to inspect."; + } + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/reactor/automation/GetReactorSignatureReactor.java b/src/prerna/reactor/automation/GetReactorSignatureReactor.java new file mode 100644 index 00000000000..560ee198ec7 --- /dev/null +++ b/src/prerna/reactor/automation/GetReactorSignatureReactor.java @@ -0,0 +1,227 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; + +import prerna.auth.User; +import prerna.auth.utils.SecurityProjectUtils; +import prerna.project.api.IProject; +import prerna.reactor.AbstractReactor; +import prerna.reactor.IReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Returns a Pixel template string and description for a custom reactor in a project. + * + *

Pixel: {@code GetReactorSignature(project=["appId"], reactor=["ReactorName"])} + * + *

Returns a JSON object with: + *

    + *
  • {@code template} - a filled Pixel call showing each param placeholder, e.g. + * {@code MyReactor(requiredKey=[""], optionalKey="")}
    • + *
  • {@code description} - the reactor's one-line description, or empty string if none
    • + *
  • {@code hasParams} - boolean, false when the reactor declares no keys
    • + *
+ * + *

On any failure (project not found, reactor not found, bad metadata) returns a minimal + * fallback object with {@code template="ReactorName()"} so the caller can still populate the field. + */ +public final class GetReactorSignatureReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GetReactorSignatureReactor.class); + + private static final String REACTOR_KEY = "reactor"; + + public GetReactorSignatureReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), REACTOR_KEY }; + this.keyRequired = new int[] { 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("You are not properly logged in."); + } + + String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); + String reactorName = this.keyValue.get(REACTOR_KEY); + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(user, projectId); + if (!SecurityProjectUtils.userCanViewProject(user, projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access."); + } + if (reactorName == null || reactorName.trim().isEmpty()) { + throw new IllegalArgumentException("A reactor name is required."); + } + reactorName = reactorName.trim(); + + JSONObject result = buildSignature(projectId, reactorName); + return new NounMetadata(result.toString(), PixelDataType.CONST_STRING, PixelOperationType.OPERATION); + } + + private static JSONObject buildSignature(String projectId, String reactorName) { + JSONObject result = new JSONObject(); + result.put("reactorName", reactorName); + + IProject project = Utility.getProject(projectId); + if (project == null) { + classLogger.warn("GetReactorSignature: project {} not found", projectId); + return fallback(result, reactorName); + } + + IReactor reactor; + try { + reactor = project.getReactor(reactorName); + } catch (Exception e) { + classLogger.warn("GetReactorSignature: could not load reactor {} from project {}", reactorName, projectId, e); + return fallback(result, reactorName); + } + if (reactor == null) { + return fallback(result, reactorName); + } + + // Description - best-effort + String description = ""; + try { + String d = reactor.getReactorDescription(); + if (d != null && !d.isBlank()) description = d.trim(); + } catch (Exception e) { + classLogger.warn("GetReactorSignature: getReactorDescription() failed for {}", reactorName, e); + } + result.put("description", description); + + // Parameter metadata via asMcpTool() + try { + JSONObject tool = reactor.asMcpTool(); + JSONObject inputSchema = tool.optJSONObject("inputSchema"); + if (inputSchema == null) { + return fallback(result, reactorName); + } + + JSONObject properties = inputSchema.optJSONObject("properties"); + JSONArray required = inputSchema.optJSONArray("required"); + + if (properties == null || properties.isEmpty()) { + result.put("template", reactorName + "()"); + result.put("hasParams", false); + return result; + } + + // Build set of required key names for O(1) lookup + Set requiredKeys = new HashSet<>(); + if (required != null) { + for (int i = 0; i < required.length(); i++) { + requiredKeys.add(required.getString(i)); + } + } + + // Build per-param metadata and required-only template + StringBuilder template = new StringBuilder(reactorName).append("("); + JSONArray params = new JSONArray(); + boolean firstRequired = true; + + // Required params first (for template ordering) + for (String key : properties.keySet()) { + if (!requiredKeys.contains(key)) continue; + if (!firstRequired) template.append(", "); + template.append(key).append("=[\"\"]"); + firstRequired = false; + + JSONObject prop = properties.optJSONObject(key); + params.put(buildParamMeta(key, prop, true)); + } + // Then optional params (not in template, but included in params list) + for (String key : properties.keySet()) { + if (requiredKeys.contains(key)) continue; + JSONObject prop = properties.optJSONObject(key); + params.put(buildParamMeta(key, prop, false)); + } + template.append(")"); + + result.put("template", template.toString()); + result.put("hasParams", !properties.isEmpty()); + result.put("params", params); + } catch (Exception e) { + classLogger.warn("GetReactorSignature: asMcpTool() failed for {}", reactorName, e); + return fallback(result, reactorName); + } + + return result; + } + + private static JSONObject buildParamMeta(String key, JSONObject prop, boolean required) { + JSONObject meta = new JSONObject(); + meta.put("name", key); + meta.put("required", required); + if (prop != null) { + String type = prop.optString("type", "string"); + meta.put("type", type); + String desc = prop.optString("description", ""); + // Suppress the default placeholder description - it adds no value + if (!desc.isBlank() && !desc.equals("No description present")) { + meta.put("description", desc); + } + } else { + meta.put("type", "string"); + } + return meta; + } + + private static JSONObject fallback(JSONObject base, String reactorName) { + base.put("template", reactorName + "()"); + base.put("description", ""); + base.put("hasParams", false); + return base; + } + + @Override + public String getReactorDescription() { + return "Returns the Pixel call template and description for a custom reactor in a project."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (REACTOR_KEY.equals(key)) return "Name of the reactor to inspect (without 'Reactor' suffix)."; + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) return "Project ID containing the reactor."; + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/reactor/automation/ListAutomationRunsReactor.java b/src/prerna/reactor/automation/ListAutomationRunsReactor.java new file mode 100644 index 00000000000..c56c017aa61 --- /dev/null +++ b/src/prerna/reactor/automation/ListAutomationRunsReactor.java @@ -0,0 +1,97 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; + +/** + * Lists automation run history for a project. + * + *

Pixel: {@code ListAutomationRuns(app=["appId"], limit=["25"])} + * + *

Reads from AUTOMATION_RUNS in the scheduler DB. + */ +public class ListAutomationRunsReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(ListAutomationRunsReactor.class); + + public ListAutomationRunsReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), ReactorKeysEnum.LIMIT.getKey() }; + this.keyRequired = new int[] { 1, 0 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); + String limitStr = this.keyValue.get(ReactorKeysEnum.LIMIT.getKey()); + int limit = parseLimit(limitStr); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanViewProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + + List> runs = AutomationDatabaseUtility.getRunsForProject(projectId, limit); + if (runs == null) { + runs = new ArrayList<>(); + } + + return new NounMetadata(runs, PixelDataType.VECTOR, PixelOperationType.OPERATION); + } + + private int parseLimit(String limitStr) { + if (limitStr == null || limitStr.isEmpty()) return AutomationConstants.DEFAULT_LIST_RUNS_LIMIT; + try { + return Integer.parseInt(limitStr.trim()); + } catch (NumberFormatException e) { + return AutomationConstants.DEFAULT_LIST_RUNS_LIMIT; + } + } + + @Override + public String getReactorDescription() { + return "Lists automation run history for a project, newest first."; + } +} diff --git a/src/prerna/reactor/automation/QuickEditAutomationReactor.java b/src/prerna/reactor/automation/QuickEditAutomationReactor.java new file mode 100644 index 00000000000..fc5ccfc952e --- /dev/null +++ b/src/prerna/reactor/automation/QuickEditAutomationReactor.java @@ -0,0 +1,243 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.PixelExecutionUtils; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.reactor.automation.utils.AutomationExecutionUtils; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; + + +/** + * Headless automation editor. The LLM provides a plain-English description of the desired change; + * this reactor chains {@code BuildAutomation} (edit mode) and {@code SaveAutomation} silently + * without opening any UI. Returns a compact summary so the LLM can narrate the result. + * + *

Use {@code EditAutomation} instead when the change is complex or when the user wants to + * review the result before saving. + * + *

Threading: {@link PixelExecutionUtils#runAndCollect} blocks the caller thread for the full + * LLM round-trip (up to {@link AutomationConstants#DEFAULT_TIMEOUT_SECONDS} seconds). Acceptable + * for MVP usage volumes - revisit under concurrent load. + * + *

Pixel: {@code QuickEditAutomation(project=["appId"], editDescription=["change the SQL to last 7 days"])} + */ +public class QuickEditAutomationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(QuickEditAutomationReactor.class); + + private static final String EDIT_DESCRIPTION_KEY = "editDescription"; + + private static final String RESULT_SUCCESS = "success"; + private static final String RESULT_NODE_COUNT = "nodeCount"; + private static final String RESULT_MESSAGE = "message"; + + public QuickEditAutomationReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), EDIT_DESCRIPTION_KEY }; + this.keyRequired = new int[] { 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(ReactorKeysEnum.PROJECT.getKey()); + String editDescription = this.keyValue.get(EDIT_DESCRIPTION_KEY); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + if (editDescription == null || editDescription.trim().isEmpty()) { + throw new IllegalArgumentException("Must provide an editDescription"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have edit access"); + } + + classLogger.info("QuickEditAutomationReactor: starting edit for project {}", projectId); + + String currentDocJson = AutomationExecutionUtils.loadAutomationDocOrEmpty(projectId); + + // BuildAutomation decodes base64 to prevent Pixel injection - same pattern used by the FE + String encodedDesc = Base64.getEncoder() + .encodeToString(editDescription.trim().getBytes(StandardCharsets.UTF_8)); + String encodedDoc = Base64.getEncoder() + .encodeToString(currentDocJson.getBytes(StandardCharsets.UTF_8)); + + String generatePixel = String.format( + "BuildAutomation(project=[\"%s\"], description=[\"%s\"], currentDoc=[\"%s\"]);", + projectId, encodedDesc, encodedDoc); + + classLogger.info("QuickEditAutomationReactor: calling BuildAutomation for project {}", projectId); + Object raw; + try { + raw = PixelExecutionUtils.runAndCollect(this.insight, generatePixel); + } catch (PixelExecutionUtils.AutomationPixelException e) { + classLogger.error("BuildAutomation pixel error for project {}", projectId, e); + throw new IllegalArgumentException("AI generation failed: " + e.getMessage()); + } + + if (raw == null) { + throw new IllegalArgumentException("BuildAutomation returned no result for project: " + projectId); + } + String generatedJson = raw instanceof String ? (String) raw : raw.toString(); + if (generatedJson.isBlank()) { + throw new IllegalArgumentException("BuildAutomation returned an empty result for project: " + projectId); + } + + Map generatedDoc = parseAndValidate(generatedJson, projectId); + + String encodedSave = Base64.getEncoder() + .encodeToString(generatedJson.getBytes(StandardCharsets.UTF_8)); + String savePixel = String.format( + "SaveAutomation(project=[\"%s\"], json=[\"%s\"]);", + projectId, encodedSave); + + classLogger.info("QuickEditAutomationReactor: calling SaveAutomation for project {}", projectId); + try { + PixelExecutionUtils.runAndCollect(this.insight, savePixel); + } catch (PixelExecutionUtils.AutomationPixelException e) { + classLogger.error("SaveAutomation pixel error for project {}", projectId, e); + throw new IllegalArgumentException("Failed to save automation: " + e.getMessage()); + } + + Map summary = buildSummary(generatedDoc); + classLogger.info("QuickEditAutomationReactor: completed for project {}", projectId); + return new NounMetadata(summary, PixelDataType.MAP, PixelOperationType.OPERATION); + } + + /** + * Parses the generated JSON and validates its structure before allowing a save. + * Throws if the document is malformed or missing the trigger node - prevents corrupt saves. + */ + @SuppressWarnings("unchecked") + private Map parseAndValidate(String json, String projectId) { + Map doc; + try { + doc = AutomationExecutionUtils.GSON.fromJson(json, AutomationExecutionUtils.MAP_TYPE); + } catch (Exception e) { + classLogger.error("BuildAutomation returned invalid JSON for project {}", projectId, e); + throw new IllegalArgumentException("AI generation returned invalid JSON - not saved."); + } + + Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); + if (graph == null) { + classLogger.error("BuildAutomation result missing 'graph' field for project {}", projectId); + throw new IllegalArgumentException("AI generation result is missing 'graph' field - not saved."); + } + + List nodes = (List) graph.get(AutomationConstants.DOC_NODES); + if (nodes == null || nodes.isEmpty()) { + classLogger.error("BuildAutomation result has no nodes for project {}", projectId); + throw new IllegalArgumentException("AI generation result has no nodes - not saved."); + } + + boolean hasTrigger = nodes.stream() + .filter(n -> n instanceof Map) + .map(n -> (Map) n) + .anyMatch(n -> AutomationConstants.NODE_TRIGGER.equals(n.get(AutomationConstants.NODE_FIELD_TYPE))); + + if (!hasTrigger) { + classLogger.error("BuildAutomation result has no trigger node for project {}", projectId); + throw new IllegalArgumentException("AI generation result has no trigger node - not saved."); + } + + return doc; + } + + /** + * Builds the summary Map returned to the LLM - concise enough to fit in context without + * sending the full automation JSON. + */ + @SuppressWarnings("unchecked") + private static Map buildSummary(Map doc) { + Map summary = new LinkedHashMap<>(); + summary.put(RESULT_SUCCESS, true); + + try { + String desc = (String) doc.getOrDefault(AutomationConstants.DOC_DESCRIPTION, ""); + Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); + List nodes = graph != null ? (List) graph.get(AutomationConstants.DOC_NODES) : null; + int nodeCount = nodes != null ? nodes.size() : 0; + summary.put(RESULT_NODE_COUNT, nodeCount); + + String message = (desc != null && !desc.isBlank()) + ? "Automation updated successfully. Description: \"" + desc.trim() + "\". " + nodeCount + " steps." + : "Automation updated successfully. " + nodeCount + " steps."; + summary.put(RESULT_MESSAGE, message); + } catch (Exception e) { + classLogger.warn("Could not build summary from generated doc", e); + summary.put(RESULT_MESSAGE, "Automation updated successfully."); + } + + return summary; + } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + return meta; + } + + @Override + public String getReactorDescription() { + return "Modifies an automation using AI generation without opening any UI. " + + "Chains BuildAutomation (edit mode) and SaveAutomation silently. " + + "Use EditAutomation instead when the user needs to review the change before saving."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) { + return "The project ID of the automation to edit."; + } else if (EDIT_DESCRIPTION_KEY.equals(key)) { + return "Plain-language description of the change to make. " + + "Example: 'Change the SQL filter to pull records from the last 7 days'. " + + "Call GetAutomationStructure first to understand the current automation structure."; + } + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/reactor/automation/RunAutomationNodeReactor.java b/src/prerna/reactor/automation/RunAutomationNodeReactor.java new file mode 100644 index 00000000000..53dea62e563 --- /dev/null +++ b/src/prerna/reactor/automation/RunAutomationNodeReactor.java @@ -0,0 +1,193 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.reactor.automation.nodes.AutomationNodeContext; +import prerna.reactor.automation.nodes.IAutomationNodeExecutor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; + +/** + * Executes a single automation node for testing/preview purposes. + * Result is NOT persisted to any run record. + * + *

Pixel: {@code RunAutomationNode(project=["appId"], nodeId=["node-id"], runId=["optional-context-run"])} + */ +public class RunAutomationNodeReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(RunAutomationNodeReactor.class); + + // Not standardized in ReactorKeysEnum - matches the local-key convention used by + // prerna.reactor.agent (e.g. GetAgentRunReactor.RUN_ID_KEY). + private static final String NODE_ID_KEY = "nodeId"; + private static final String RUN_ID_KEY = "runId"; + + public RunAutomationNodeReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), NODE_ID_KEY, RUN_ID_KEY }; + this.keyRequired = new int[] { 1, 1, 0 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + String nodeId = this.keyValue.get(this.keysToGet[1]); + String contextRunId = this.keyValue.get(this.keysToGet[2]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + if (nodeId == null || nodeId.isEmpty()) { + throw new IllegalArgumentException("Must provide a node id"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + + Map node = findNode(projectId, nodeId); + if (node == null) { + throw new IllegalArgumentException("Node not found in automation: " + nodeId); + } + + Map scope = buildScope(projectId, contextRunId); + Map configMap = AutomationExecutionUtils.loadConfig(projectId); + + long startMs = System.currentTimeMillis(); + try { + String type = (String) node.get(AutomationConstants.NODE_FIELD_TYPE); + Object rawOutput; + + if (AutomationConstants.NODE_TRIGGER.equals(type)) { + rawOutput = scope.get(AutomationConstants.SCOPE_TRIGGERED_AT); + } else { + IAutomationNodeExecutor executor = IAutomationNodeExecutor.EXECUTORS.get(type); + if (executor == null) { + throw new IllegalArgumentException("Unsupported node type: " + type); + } + AutomationNodeContext ctx = new AutomationNodeContext( + AutomationConstants.TEST_RUN_ID, projectId, node, scope, configMap, + this.insight, new AtomicBoolean(false)); + rawOutput = executor.execute(ctx); + } + + @SuppressWarnings("unchecked") + Map transformConfig = (Map) node.get(AutomationConstants.NODE_FIELD_OUTPUT_TRANSFORM); + String transformed = AutomationExecutionUtils.applyOutputTransform(rawOutput, transformConfig); + long durationMs = System.currentTimeMillis() - startMs; + Map result = new HashMap<>(); + result.put(AutomationConstants.NODE_ID, nodeId); + result.put(AutomationConstants.STATUS, AutomationConstants.NODE_STATUS_SUCCESS); + result.put(AutomationConstants.DURATION_MS, durationMs); + result.put(AutomationConstants.OUTPUT_PREVIEW, transformed); + result.put(AutomationConstants.OUTPUT_VALUE, transformed); + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + + } catch (Exception e) { + long durationMs = System.currentTimeMillis() - startMs; + classLogger.error("Test run of node {} failed: {}", nodeId, e.getMessage(), e); + + Map result = new HashMap<>(); + result.put(AutomationConstants.NODE_ID, nodeId); + result.put(AutomationConstants.STATUS, AutomationConstants.NODE_STATUS_FAILED); + result.put(AutomationConstants.DURATION_MS, durationMs); + result.put(AutomationConstants.ERROR_MESSAGE, e.getMessage()); + return new NounMetadata(result, PixelDataType.MAP, PixelOperationType.OPERATION); + } + } + + @SuppressWarnings("unchecked") + private static Map findNode(String projectId, String nodeId) { + Map doc = AutomationExecutionUtils.loadAutomationDoc(projectId); + Map graph = (Map) doc.get(AutomationConstants.DOC_GRAPH); + List> nodes = (List>) graph.get(AutomationConstants.DOC_NODES); + if (nodes != null) { + for (Map node : nodes) { + if (nodeId.equals(node.get(AutomationConstants.NODE_FIELD_ID))) return node; + } + } + return null; + } + + private Map buildScope(String projectId, String contextRunId) { + Map scope = AutomationExecutionUtils.buildInitialScope(null, this.insight.getUser()); + + if (contextRunId != null && !contextRunId.isEmpty()) { + // Scope the context run to this project - otherwise a user could pull node outputs + // (potentially containing other apps' secrets/data) from a run belonging to a + // project they don't have access to by passing an arbitrary runId. + Map contextRunDetail = AutomationDatabaseUtility.getRunDetail(contextRunId); + if (contextRunDetail == null || !projectId.equals(contextRunDetail.get(AutomationConstants.PROJECT_ID))) { + throw new IllegalArgumentException("Run not found: " + contextRunId); + } + + List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(contextRunId); + for (Map output : nodeOutputs) { + String status = (String) output.get(AutomationConstants.STATUS); + if (AutomationConstants.NODE_STATUS_SUCCESS.equals(status)) { + String outputVar = (String) output.get(AutomationConstants.OUTPUT_VAR); + if (outputVar != null && !outputVar.isEmpty()) { + Object value = output.get(AutomationConstants.OUTPUT_VALUE); + scope.put(outputVar, value != null ? value.toString() : ""); + } + } + } + } + return scope; + } + + @Override + public String getReactorDescription() { + return "Executes a single automation node in isolation for testing - result is not persisted."; + } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + // Node executors can perform real side effects (DB writes, storage uploads/deletes, + // arbitrary pixel execution) - requires explicit human confirmation. + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + return meta; + } +} diff --git a/src/prerna/reactor/automation/SaveAutomationConfigReactor.java b/src/prerna/reactor/automation/SaveAutomationConfigReactor.java new file mode 100644 index 00000000000..7ba70edcc43 --- /dev/null +++ b/src/prerna/reactor/automation/SaveAutomationConfigReactor.java @@ -0,0 +1,197 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +import java.io.File; +import java.io.IOException; +import java.util.Base64; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.reflect.TypeToken; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.util.Utility; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.AssetUtility; + +public class SaveAutomationConfigReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(SaveAutomationConfigReactor.class); + + private static final java.lang.reflect.Type LIST_OF_MAP_TYPE = + new TypeToken>>() {}.getType(); + + public SaveAutomationConfigReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), ReactorKeysEnum.CONFIG.getKey() }; + this.keyRequired = new int[] { 1, 0 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + String configEncoded = this.keyValue.get(this.keysToGet[1]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have edit access"); + } + + String config; + if (configEncoded == null || configEncoded.isEmpty()) { + config = AutomationConstants.EMPTY_JSON_ARRAY; + } else { + try { + config = new String(Base64.getDecoder().decode(configEncoded.trim()), StandardCharsets.UTF_8); + } catch (Exception e) { + // configEncoded was not Base64 - treat as raw JSON and validate it parses below + config = configEncoded; + } + } + // Validate JSON is parseable before writing anything + try { + AutomationExecutionUtils.GSON.fromJson(config, LIST_OF_MAP_TYPE); + } catch (Exception e) { + throw new IllegalArgumentException("config must be valid JSON or Base64-encoded JSON"); + } + + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File configFile = Paths.get(portalsFolder, AutomationConstants.AUTOMATION_CONFIG_FILE_NAME).toFile(); + String normalizedConfigPath = Utility.normalizePath(configFile.getAbsolutePath()); + if (!normalizedConfigPath.startsWith(portalsFolder)) { + throw new IllegalArgumentException("Invalid file path"); + } + + // GetAutomationConfig masks sensitive values as SENSITIVE_MASK. + // Restore the real values from disk so a config round-trip cannot silently destroy them. + config = restoreMaskedSensitiveValues(config, configFile); + + try { + configFile.getParentFile().mkdirs(); + Files.writeString(configFile.toPath(), config, StandardCharsets.UTF_8); + } catch (IOException e) { + classLogger.error("Error saving automation config for project {}", projectId, e); + throw new IllegalArgumentException("Unable to save automation config: " + e.getMessage()); + } + + SecurityProjectUtils.updateProjectLastEditedDate(projectId); + return new NounMetadata(true, PixelDataType.BOOLEAN, PixelOperationType.OPERATION); + } + + /** + * For every incoming entry marked {@code sensitive} whose value is still the + * {@link AutomationConstants#SENSITIVE_MASK} placeholder, restore the real value from the + * existing on-disk config (matched by {@code key}). Prevents a save from the UI - which + * only ever received the masked value - from overwriting the stored secret. + */ + @SuppressWarnings("unchecked") + private String restoreMaskedSensitiveValues(String incomingJson, File existingFile) { + if (existingFile == null || !existingFile.exists()) { + return incomingJson; + } + try { + List> incoming = AutomationExecutionUtils.GSON.fromJson(incomingJson, + LIST_OF_MAP_TYPE); + String existingJson = Files.readString(existingFile.toPath(), StandardCharsets.UTF_8); + List> existing = AutomationExecutionUtils.GSON.fromJson(existingJson, + LIST_OF_MAP_TYPE); + if (incoming == null || existing == null || existing.isEmpty()) { + return incomingJson; + } + + Map existingValueByKey = new HashMap<>(); + for (Map e : existing) { + existingValueByKey.put(String.valueOf(e.get(AutomationConstants.CONFIG_ENTRY_KEY)), e.get(AutomationConstants.CONFIG_ENTRY_VALUE)); + } + + boolean restoredAny = false; + for (Map entry : incoming) { + if (Boolean.TRUE.equals(entry.get(AutomationConstants.CONFIG_ENTRY_SENSITIVE)) + && AutomationConstants.SENSITIVE_MASK.equals(entry.get(AutomationConstants.CONFIG_ENTRY_VALUE))) { + Object real = existingValueByKey.get(String.valueOf(entry.get(AutomationConstants.CONFIG_ENTRY_KEY))); + if (real != null) { + entry.put(AutomationConstants.CONFIG_ENTRY_VALUE, real); + restoredAny = true; + } + } + } + return restoredAny ? AutomationExecutionUtils.GSON.toJson(incoming) : incomingJson; + } catch (Exception e) { + // Never let a merge problem overwrite good secrets - keep the existing file untouched. + classLogger.warn("Could not merge sensitive automation config; keeping existing file. Cause: {}", + e.getMessage()); + try { + return Files.readString(existingFile.toPath(), StandardCharsets.UTF_8); + } catch (IOException io) { + return incomingJson; + } + } + } + + @Override + public String getReactorDescription() { + return "Saves the automation config (key/value env vars and secrets) for a project."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) { + return "The project ID whose automation config should be saved."; + } else if (ReactorKeysEnum.CONFIG.getKey().equals(key)) { + return "Base64-encoded JSON array of config entries (key, value, sensitive flag)."; + } + return super.getDescriptionForKey(key); + } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + // Overwrites saved config values/secrets — requires explicit human confirmation. + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + return meta; + } +} diff --git a/src/prerna/reactor/automation/SaveAutomationReactor.java b/src/prerna/reactor/automation/SaveAutomationReactor.java new file mode 100644 index 00000000000..792b16617e4 --- /dev/null +++ b/src/prerna/reactor/automation/SaveAutomationReactor.java @@ -0,0 +1,164 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import java.io.File; +import java.io.IOException; +import java.util.Base64; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.cluster.util.ClusterUtil; +import prerna.project.api.IProject; +import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.AssetUtility; +import prerna.util.Utility; +import prerna.util.git.GitRepoUtils; + +public class SaveAutomationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(SaveAutomationReactor.class); + + public SaveAutomationReactor() { + this.keysToGet = new String[]{ ReactorKeysEnum.PROJECT.getKey(), ReactorKeysEnum.JSON.getKey() }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = this.keyValue.get(this.keysToGet[0]); + String jsonEncoded = this.keyValue.get(this.keysToGet[1]); + + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + if (jsonEncoded == null || jsonEncoded.isEmpty()) { + throw new IllegalArgumentException("Must provide automation JSON"); + } + + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have edit access"); + } + + String json; + try { + json = new String(Base64.getDecoder().decode(jsonEncoded), StandardCharsets.UTF_8); + } catch (Exception e) { + json = jsonEncoded; + } + AutomationDefinitionValidator.parseAndValidate(json); + + IProject project = Utility.getProject(projectId); + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File automationFile = Paths.get(portalsFolder, AutomationConstants.AUTOMATION_FILE_NAME).toFile(); + String normalizedPath = Utility.normalizePath(automationFile.getAbsolutePath()); + if (!normalizedPath.startsWith(portalsFolder)) { + throw new IllegalArgumentException("Invalid file path"); + } + + try { + automationFile.getParentFile().mkdirs(); + Files.writeString(automationFile.toPath(), json, StandardCharsets.UTF_8); + } catch (IOException e) { + classLogger.error("Error saving automation JSON for project {}", projectId, e); + throw new IllegalArgumentException("Unable to save automation: " + e.getMessage()); + } + + if (project == null) { + classLogger.warn("Project {} not found in registry - skipping git commit and cluster push", projectId); + SecurityProjectUtils.updateProjectLastEditedDate(projectId); + AutomationMcpSync.syncTriggerAutomationTool(null, projectId, this.insight.getUser(), json); + return new NounMetadata(true, PixelDataType.BOOLEAN, PixelOperationType.OPERATION); + } + + List files = new ArrayList<>(); + files.add(automationFile.getAbsolutePath()); + String versionFolder = AssetUtility.getProjectVersionFolder(project.getProjectName(), projectId); + try { + GitRepoUtils.addSpecificFiles(versionFolder, files); + GitRepoUtils.commitAddedFiles(versionFolder, "Update automation graph", this.insight.getUser()); + } catch (Exception e) { + classLogger.warn("Git commit failed for automation save", e); + } + + if (ClusterUtil.IS_CLUSTER) { + try { + ClusterUtil.pushProjectFolder(project, versionFolder); + } catch (Exception e) { + classLogger.warn("Cluster push failed", e); + } + } + + SecurityProjectUtils.updateProjectLastEditedDate(projectId); + + // Keep this project's own MCP tool catalog in sync with every save, so the automation is + // always discoverable as a "TriggerAutomation" tool without a separate manual step. + AutomationMcpSync.syncTriggerAutomationTool(project, projectId, this.insight.getUser(), json); + + return new NounMetadata(true, PixelDataType.BOOLEAN, PixelOperationType.OPERATION); + } + + @Override + public String getReactorDescription() { + return "Saves the automation graph (automation.json) for a project."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) { + return "The project ID of the automation to save."; + } else if (ReactorKeysEnum.JSON.getKey().equals(key)) { + return "Base64-encoded JSON document representing the automation graph (nodes, edges, transforms)."; + } + return super.getDescriptionForKey(key); + } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + // Overwrites the saved automation graph and commits it to source control - + // requires explicit human confirmation. + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + return meta; + } +} diff --git a/src/prerna/reactor/automation/TriggerAutomationReactor.java b/src/prerna/reactor/automation/TriggerAutomationReactor.java new file mode 100644 index 00000000000..3e7500e2ed3 --- /dev/null +++ b/src/prerna/reactor/automation/TriggerAutomationReactor.java @@ -0,0 +1,227 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.reactor.agent.mcp.MCPUtility.MCPDisplayOption; +import prerna.reactor.agent.mcp.MCPUtility.MCPExecution; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; + +/** + * Manually triggers an automation run for a project. Validates access, claims the single-run slot, + * runs the automation synchronously on the calling thread (expected to be a virtual thread from + * the platform's {@code runPixelAsync} endpoint), and returns the completed run result. + * + *

Pixel: {@code TriggerAutomation(project=["appId"])} + */ +public class TriggerAutomationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(TriggerAutomationReactor.class); + + public TriggerAutomationReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.PROJECT.getKey(), AutomationConstants.AUTOMATION_INPUTS_KEY, AutomationConstants.AUTOMATION_TRIGGER_TYPE_KEY }; + this.keyRequired = new int[] { 1, 0, 0 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String projectId = getProjectId(); + String userId = getUserId(); + String runId = UUID.randomUUID().toString(); + + // Validate the automation has runnable steps BEFORE claiming the run slot, + // so a bad automation never leaves a stale active-run record. + AutomationDefinitionValidator.ValidatedDefinition definition = + AutomationDefinitionValidator.validate(AutomationExecutionUtils.loadAutomationDoc(projectId)); + List> ordered = definition.getExecutionOrder(); + long nonTriggerCount = ordered.stream() + .filter(n -> !AutomationConstants.NODE_TRIGGER.equals(n.get(AutomationConstants.NODE_FIELD_TYPE))) + .count(); + if (nonTriggerCount == 0) { + throw new IllegalArgumentException("Automation has no steps to run. Add at least one step before running."); + } + + if (!AutomationDatabaseUtility.claimActiveRun(projectId, runId)) { + String activeRun = AutomationDatabaseUtility.getActiveRun(projectId); + throw new IllegalArgumentException( + "Automation already has an active run: " + activeRun + + ". Wait for it to complete or cancel it before starting a new run."); + } + + boolean runStarted = false; + try { + Map configMap = AutomationExecutionUtils.loadConfig(projectId); + + @SuppressWarnings("unchecked") + Map inputsMap = this.getMap(AutomationConstants.AUTOMATION_INPUTS_KEY); + AutomationExecutionUtils.applyPlaygroundInputs(ordered, inputsMap); + definition = AutomationDefinitionValidator.validate(definition.getDocument()); + ordered = definition.getExecutionOrder(); + + String triggerType = this.keyValue.get(AutomationConstants.AUTOMATION_TRIGGER_TYPE_KEY); + if (triggerType == null || triggerType.isBlank()) { + triggerType = AutomationConstants.TRIGGER_MANUAL; + } else if (!AutomationConstants.TRIGGER_MANUAL.equals(triggerType) + && !AutomationConstants.TRIGGER_PLAYGROUND.equals(triggerType)) { + classLogger.warn("Unknown triggerType '{}' for run {}, defaulting to MANUAL", triggerType, runId); + triggerType = AutomationConstants.TRIGGER_MANUAL; + } + AutomationDatabaseUtility.insertRun(runId, projectId, AutomationConstants.DEFAULT_AUTOMATION_ID, + definition.getVersion(), definition.getHash(), definition.getSnapshot(), + triggerType, ordered.size(), userId); + AutomationDatabaseUtility.insertAllNodeOutputs(runId, ordered); + + classLogger.info("Automation run {} starting for project {}", runId, projectId); + runStarted = true; + Map finalScope = AutomationRunEngine.run(runId, projectId, ordered, configMap, this.insight); + + // Build final result in the same shape as GetAutomationRunReactor + Map runDetail = AutomationDatabaseUtility.getRunDetail(runId); + List> nodeOutputs = AutomationDatabaseUtility.getNodeOutputsForRun(runId); + List> nodeResults = AutomationDatabaseUtility.buildNodeResults(nodeOutputs); + int completedCount = 0; + for (Map nodeResult : nodeResults) { + if (AutomationConstants.NODE_STATUS_SUCCESS.equals(nodeResult.get(AutomationConstants.STATUS))) { + completedCount++; + } + } + if (runDetail == null) { + runDetail = new HashMap<>(); + runDetail.put(AutomationConstants.RUN_ID, runId); + runDetail.put(AutomationConstants.PROJECT_ID, projectId); + } + runDetail.put(AutomationConstants.RESULT_NODE_RESULTS, nodeResults); + + boolean runSucceeded = AutomationConstants.STATUS_SUCCESS.equals(runDetail.get(AutomationConstants.STATUS)); + // Short summary shown in the sidebar UI. + String summary = runSucceeded + ? AutomationExecutionUtils.buildSummaryMessage(definition.getDocument(), finalScope, configMap, + completedCount, ordered.size()) + : buildFailureSummary(runDetail); + runDetail.put(AutomationConstants.RESULT_SUMMARY, summary); + AutomationDatabaseUtility.updateRunSummary(runId, summary); + + // Enriched context sent to the LLM as the MCP tool response, so it can describe + // what each step actually did rather than just echoing the node count. + if (runSucceeded) { + StringBuilder llmContext = new StringBuilder(summary); + for (Map step : nodeResults) { + String label = (String) step.get(AutomationConstants.NODE_LABEL); + String preview = (String) step.get(AutomationConstants.OUTPUT_PREVIEW); + if (preview != null && !preview.isBlank()) { + llmContext.append("\n- ").append(label).append(": ").append(preview); + } + } + runDetail.put(AutomationConstants.RESULT_LLM_CONTEXT, llmContext.toString()); + } + + return new NounMetadata(runDetail, PixelDataType.MAP, PixelOperationType.OPERATION); + + } catch (Exception e) { + classLogger.error("Automation run setup failed for project {}, run {}", projectId, runId, e); + // Only release if AutomationRunEngine.run() was never called - once it starts, + // its own finally block handles releaseActiveRun. + if (!runStarted) { + AutomationDatabaseUtility.releaseActiveRun(projectId, runId); + } + if (e instanceof RuntimeException re) throw re; + throw new RuntimeException(e); + } + } + + // -- Helpers ------------------------------------------------------------------- + + /** Null-safe failure summary - {@code FAILED_NODE_ID}/{@code ERROR_MESSAGE} may be absent. */ + private static String buildFailureSummary(Map runDetail) { + Object failedNodeId = runDetail.get(AutomationConstants.FAILED_NODE_ID); + Object errorMessage = runDetail.get(AutomationConstants.ERROR_MESSAGE); + return "Automation failed at node " + (failedNodeId != null ? failedNodeId : "unknown") + + ": " + (errorMessage != null ? errorMessage : "no error details available"); + } + + private String getProjectId() { + String projectId = this.keyValue.get(this.keysToGet[0]); + if (projectId == null || projectId.isEmpty()) { + throw new IllegalArgumentException("Must provide a project id"); + } + projectId = SecurityProjectUtils.testUserProjectIdForAlias(this.insight.getUser(), projectId); + if (!SecurityProjectUtils.userCanEditProject(this.insight.getUser(), projectId)) { + throw new IllegalArgumentException("Project does not exist or user does not have access"); + } + return projectId; + } + + private String getUserId() { + if (this.insight.getUser() != null && this.insight.getUser().getPrimaryLoginToken() != null) { + return this.insight.getUser().getPrimaryLoginToken().getId(); + } + return AutomationConstants.SYSTEM_USER_ID; + } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPExecution.ASK.getValue()); + meta.put(MCPUtility.UI_DISPLAY_LOCATION, MCPDisplayOption.SIDEBAR.getValue()); + // Same "system app" resourceURI scheme used by the Playwright browser-sockets tool + // (see PlaywrightMCPToolBuilder) - resolved client-side to the automation workspace's + // own build output (../../automation-workspace/dist/) so the exact same UI renders + // whether embedded directly in the client app or iframed as an MCP sidebar tool. + meta.put(MCPUtility.UI_RESOURCE_URI, "system://automation-workspace/?readOnly=1"); + return meta; + } + + @Override + public String getReactorDescription() { + return "Manually triggers an automation run for the given project and returns a per-step summary once complete."; + } + + @Override + protected String getDescriptionForKey(String key) { + if (ReactorKeysEnum.PROJECT.getKey().equals(key)) return "The project (app) ID or alias to run the automation for."; + if (AutomationConstants.AUTOMATION_INPUTS_KEY.equals(key)) return "Optional map of playground-supplied values to inject into automation node fields before running. Keys are parameter names from the automation's MCP tool schema."; + if (AutomationConstants.AUTOMATION_TRIGGER_TYPE_KEY.equals(key)) return "Caller-supplied trigger source. Defaults to MANUAL when omitted; MCP/playground callers pass PLAYGROUND."; + return super.getDescriptionForKey(key); + } +} diff --git a/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java new file mode 100644 index 00000000000..9e0bbbbba8c --- /dev/null +++ b/src/prerna/reactor/automation/nodes/AppEngineNodeExecutor.java @@ -0,0 +1,104 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityProjectUtils; +import prerna.om.ThreadStore; +import prerna.project.api.IProject; +import prerna.reactor.automation.AutomationConstants; +import prerna.reactor.automation.utils.AutomationExecutionUtils; +import prerna.reactor.automation.utils.PixelExecutionUtils; +import prerna.util.Utility; + +/** + * Executor for {@code app}-type nodes. Runs an arbitrary pixel expression, optionally + * inside a specific app's project context. + * + *

Config fields (from {@code node.config}): + *

    + *
  • {@code pixel} (required) — the pixel expression to run; supports {@code ${var}} substitution
    • + *
  • {@code appId} (optional) — if set, the pixel runs inside this project's insight context
    • + *
+ * + *

When {@code appId} is provided the project context overrides are set on {@link ThreadStore} + * before execution. {@link PixelExecutionUtils#runAndCollect} snapshots the caller's thread + * context (including these overrides) before submitting to the timeout thread, so the context + * propagates correctly even under timeout enforcement. + * + *

The caller must have edit access to {@code appId} — this executor runs an arbitrary pixel + * expression in that project's context, so a view-only check would let any automation editor + * reach into projects they cannot otherwise modify. + */ +public final class AppEngineNodeExecutor implements IAutomationNodeExecutor { + + private static final Logger classLogger = LogManager.getLogger(AppEngineNodeExecutor.class); + + @Override + public Object execute(AutomationNodeContext ctx) throws Exception { + Map config = ctx.config(); + String nodeLabel = ctx.nodeLabel(); + Map scope = ctx.scope(); + Map configMap = ctx.configMap(); + + String pixel = NodeConfigHelper.required(config, AutomationConstants.CONFIG_PIXEL, nodeLabel); + String appId = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_APP_ID); + String resolvedPixel = AutomationExecutionUtils.resolve(pixel, scope, configMap); + String resolvedAppId = appId != null ? AutomationExecutionUtils.resolve(appId, scope, configMap) : null; + + classLogger.debug("App-engine node \"{}\" executing pixel in appId={}", nodeLabel, resolvedAppId != null ? resolvedAppId : "caller context"); + + int timeoutMs = AutomationExecutionUtils.getNodeTimeout(ctx.node()); + if (resolvedAppId != null && !resolvedAppId.isBlank()) { + resolvedAppId = SecurityProjectUtils.testUserProjectIdForAlias(ctx.insight().getUser(), resolvedAppId); + if (!SecurityProjectUtils.userCanEditProject(ctx.insight().getUser(), resolvedAppId)) { + throw new IllegalArgumentException( + "App node \"" + nodeLabel + "\": project does not exist or user does not have access: " + resolvedAppId); + } + IProject project = Utility.getProject(resolvedAppId); + if (project == null) { + throw new IllegalArgumentException( + "App node \"" + nodeLabel + "\": project not found: " + resolvedAppId); + } + ThreadStore.setContextProjectIdOverride(resolvedAppId); + ThreadStore.setContextProjectNameOverride(project.getProjectName()); + try { + return PixelExecutionUtils.runAndCollect(ctx.insight(), resolvedPixel, timeoutMs); + } finally { + ThreadStore.clearContextProjectOverride(); + } + } + + return PixelExecutionUtils.runAndCollect(ctx.insight(), resolvedPixel, timeoutMs); + } + +} diff --git a/src/prerna/reactor/automation/nodes/AutomationNodeContext.java b/src/prerna/reactor/automation/nodes/AutomationNodeContext.java new file mode 100644 index 00000000000..09bc55bb08a --- /dev/null +++ b/src/prerna/reactor/automation/nodes/AutomationNodeContext.java @@ -0,0 +1,91 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import prerna.om.Insight; +import prerna.reactor.automation.AutomationConstants; + +/** + * Param bundle passed to every {@link IAutomationNodeExecutor}. + * + *

Consolidates all per-node execution inputs so executor implementations have a single, + * typed object to work with rather than long parameter lists. + * + *

Note: {@code scope} is intentionally mutable and shared across all node executions within + * one run. Each node may add output variables to {@code scope} after it completes successfully, + * making those values available to subsequent nodes via {@code ${varName}} substitution. + * + * @param runId the automation run ID (use {@code "test"} for single-node test runs) + * @param projectId the owning project UUID + * @param node the raw node definition map from {@code automation.json} + * @param scope mutable variable scope — string key→value pairs resolved by {@code ${varName}}; + * shared across all nodes in the run and updated after each node completes + * @param configMap project automation config key→value pairs resolved by {@code ${config.KEY}} + * @param insight caller's {@link Insight} context (carries user, session, pixel engine) + * @param cancelFlag shared flag the executor checks to honour mid-node cancellation requests + */ +public record AutomationNodeContext( + String runId, + String projectId, + Map node, + Map scope, + Map configMap, + Insight insight, + AtomicBoolean cancelFlag) { + + public String nodeId() { + return (String) node.get(AutomationConstants.NODE_FIELD_ID); + } + + public String nodeLabel() { + Object label = node.get(AutomationConstants.NODE_FIELD_LABEL); + return label != null ? label.toString() : AutomationConstants.UNNAMED_NODE_LABEL; + } + + public String nodeType() { + return (String) node.get(AutomationConstants.NODE_FIELD_TYPE); + } + + /** + * Returns the node's config map, or an empty map if absent. + * + *

The cast to {@code Map} is safe: Gson always deserializes JSON objects + * as {@code Map}, and the automation document is loaded exclusively via Gson + * in {@link prerna.reactor.automation.AutomationExecutionUtils#loadAutomationDoc}. + */ + @SuppressWarnings("unchecked") + public Map config() { + Object config = node.get(AutomationConstants.NODE_FIELD_CONFIG); + return config instanceof Map ? (Map) config : Map.of(); + } +} diff --git a/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java new file mode 100644 index 00000000000..85b85821f82 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/DatabaseEngineNodeExecutor.java @@ -0,0 +1,84 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.reactor.automation.AutomationConstants; +import prerna.reactor.automation.utils.AutomationExecutionUtils; +import prerna.reactor.automation.utils.PixelExecutionUtils; + +/** + * Executor for {@code database-engine} nodes. Runs a SQL query against a configured database engine + * using the {@code SqlQuery} reactor — the same engine abstraction used elsewhere in the platform. + * + *

Config fields: + *

    + *
  • {@code engineId} (required) — UUID or alias of the target database engine
    • + *
  • {@code expression} (required) — SQL expression; supports {@code ${var}} substitution
    • + *
  • {@code operation} (optional) — {@code "read"} (default) or {@code "write"}; informational + * only since {@code SqlQuery} auto-detects SELECT vs DML from the SQL text
    • + *
  • {@code limit} (optional) — max rows for SELECT results; default 50
    • + *
+ */ +public final class DatabaseEngineNodeExecutor implements IAutomationNodeExecutor { + + private static final Logger classLogger = LogManager.getLogger(DatabaseEngineNodeExecutor.class); + + @Override + public Object execute(AutomationNodeContext ctx) throws Exception { + Map config = ctx.config(); + String nodeLabel = ctx.nodeLabel(); + Map scope = ctx.scope(); + Map configMap = ctx.configMap(); + + String engineId = NodeConfigHelper.required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); + String sql = NodeConfigHelper.required(config, AutomationConstants.CONFIG_EXPRESSION, nodeLabel); + String operation = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_READ); + int limit = NodeConfigHelper.optionalInt(config, AutomationConstants.CONFIG_LIMIT, AutomationConstants.DEFAULT_DB_QUERY_LIMIT); + + String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); + String resolvedSql = AutomationExecutionUtils.resolve(sql, scope, configMap); + + classLogger.debug("Database-engine node \"{}\" executing operation={} via engine {}", nodeLabel, operation, resolvedEngineId); + + // Escape double quotes in the SQL and engine id for the pixel string literal, then + // delegate to SqlQuery which uses the engine abstraction (HardSelectQueryStruct) and + // enforces the caller's database-level permissions automatically. + String escapedEngineId = resolvedEngineId.replace("\"", "\\\""); + String escapedSql = resolvedSql.replace("\"", "\\\""); + String pixel = "SqlQuery(database=[\"" + escapedEngineId + "\"], query=[\"" + escapedSql + "\"], limit=[" + limit + "]);"; + + int timeout = AutomationExecutionUtils.getNodeTimeout(ctx.node()); + return PixelExecutionUtils.runAndCollect(ctx.insight(), pixel, timeout); + } + +} diff --git a/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java new file mode 100644 index 00000000000..a93641f5aa5 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/FunctionEngineNodeExecutor.java @@ -0,0 +1,101 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityQueryUtils; +import prerna.engine.api.IFunctionEngine; +import prerna.reactor.automation.AutomationConstants; +import prerna.reactor.automation.utils.AutomationExecutionUtils; +import prerna.util.Utility; + +/** + * Executes a function-engine automation node. Delegates to the platform + * {@link prerna.engine.api.IFunctionEngine} API to run arbitrary server-side functions. + * + *

Config fields (from {@code node.config}): + *

    + *
  • {@code engineId} (required) — UUID or alias of the target function engine
    • + *
  • {@code params} (optional) — JSON object of parameters to pass to the function; supports + * {@code ${var}} substitution; defaults to {@code {}}
    • + *
+ * + *

Requires edit access to {@code engineId} because function execution runs arbitrary + * server-side code, which is a mutating/write operation. + */ +public final class FunctionEngineNodeExecutor implements IAutomationNodeExecutor { + + private static final Logger classLogger = LogManager.getLogger(FunctionEngineNodeExecutor.class); + + @Override + public Object execute(AutomationNodeContext ctx) throws Exception { + Map config = ctx.config(); + String nodeLabel = ctx.nodeLabel(); + Map scope = ctx.scope(); + Map configMap = ctx.configMap(); + + String engineId = NodeConfigHelper.required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); + String params = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_PARAMS, AutomationConstants.EMPTY_JSON_OBJECT); + + String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); + String resolvedParams = AutomationExecutionUtils.resolve(params, scope, configMap); + + resolvedEngineId = SecurityQueryUtils.testUserEngineIdForAlias(ctx.insight().getUser(), resolvedEngineId); + if (!SecurityEngineUtils.userCanEditEngine(ctx.insight().getUser(), resolvedEngineId)) { + throw new IllegalArgumentException( + "Function-engine node \"" + nodeLabel + "\": engine does not exist or user does not have edit access: " + resolvedEngineId); + } + + IFunctionEngine engine = Utility.getFunctionEngine(resolvedEngineId); + if (engine == null) { + throw new IllegalArgumentException("Function-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); + } + + classLogger.debug("Function-engine node \"{}\" executing via engine {}", nodeLabel, resolvedEngineId); + Map paramMap = parseParams(resolvedParams, nodeLabel); + return engine.execute(paramMap); + } + + @SuppressWarnings("unchecked") + private static Map parseParams(String json, String nodeLabel) { + if (json == null || json.isBlank()) return Map.of(); + try { + Map parsed = AutomationExecutionUtils.GSON.fromJson(json, + AutomationExecutionUtils.MAP_TYPE); + return parsed != null ? parsed : Map.of(); + } catch (Exception e) { + throw new IllegalArgumentException("Function-engine node \"" + nodeLabel + "\": params is not valid JSON: " + e.getMessage(), e); + } + } + +} diff --git a/src/prerna/reactor/automation/nodes/IAutomationNodeExecutor.java b/src/prerna/reactor/automation/nodes/IAutomationNodeExecutor.java new file mode 100644 index 00000000000..1f9aacd37a9 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/IAutomationNodeExecutor.java @@ -0,0 +1,86 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +import prerna.reactor.automation.AutomationConstants; + +/** + * Executes one automation node's operation, given all the context that node type needs. + * + *

One concrete implementation per {@code node.type} value (e.g. {@code WaitNodeExecutor}, + * {@code DatabaseEngineNodeExecutor}), resolved via {@link #EXECUTORS} instead of the + * previous {@code if/else} chain keyed on {@code type}. + * + *

Mirrors the shape already established in this codebase for "one conceptual operation, many + * type-specific implementations, resolved by a type key" - see + * {@link prerna.engine.api.IModelEngine} (resolved via {@code Utility.getModel(engineId)}) and + * {@link prerna.engine.api.IMCP}. + * + *

Implementations should be stateless and safe to share as a single static instance across + * concurrent runs - all per-run/per-node state is passed in via {@link AutomationNodeContext}, not + * held on the executor instance. + */ +public interface IAutomationNodeExecutor { + + /** + * Shared registry of stateless node executor instances, keyed by {@code node.type} value. + * Used by both {@link prerna.reactor.automation.AutomationRunEngine} and + * {@link prerna.reactor.automation.RunAutomationNodeReactor}. + * + *

Static interface fields are implicitly {@code public static final} in Java. + * + * + */ + Map EXECUTORS = Map.of( + AutomationConstants.NODE_WAIT, new WaitNodeExecutor(), + AutomationConstants.NODE_DATABASE_ENGINE, new DatabaseEngineNodeExecutor(), + AutomationConstants.NODE_MODEL_ENGINE, new ModelEngineNodeExecutor(), + AutomationConstants.NODE_VECTOR_ENGINE, new VectorEngineNodeExecutor(), + AutomationConstants.NODE_STORAGE_ENGINE, new StorageEngineNodeExecutor(), + AutomationConstants.NODE_FUNCTION_ENGINE, new FunctionEngineNodeExecutor(), + AutomationConstants.NODE_APP, new AppEngineNodeExecutor() + ); + + /** + * Executes this node's operation and returns its raw output - the same shape previously + * returned by each {@code executeXNode} method (a String, a JSON string, or a + * {@code Map} that the caller will serialize). The caller + * ({@link prerna.reactor.automation.AutomationRunEngine#executeSingleNode}) is responsible for + * applying the node's output transform, generating the preview, and checkpointing + * success/failure to the database - executors should not do any of that themselves. + * + * @param ctx the node's execution context - node definition, scope, config, and callbacks + * for recursing into sibling/child nodes where the node type requires it + * @return the node's raw output + * @throws Exception on any failure - the caller catches and records it as a failed node, + * following the same contract the previous {@code executeXNode} methods had + */ + Object execute(AutomationNodeContext ctx) throws Exception; +} diff --git a/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java new file mode 100644 index 00000000000..6fbfcf49daa --- /dev/null +++ b/src/prerna/reactor/automation/nodes/ModelEngineNodeExecutor.java @@ -0,0 +1,117 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityQueryUtils; +import prerna.engine.api.IModelEngine; +import prerna.engine.impl.model.responses.AskModelEngineResponse; +import prerna.engine.impl.model.responses.EmbeddingsModelEngineResponse; +import prerna.reactor.automation.AutomationConstants; +import prerna.reactor.automation.utils.AutomationExecutionUtils; +import prerna.util.Utility; + +/** + * Executes a model-engine automation node. Supports LLM ({@code llm}) and embeddings + * ({@code embeddings}) operations via the platform {@link prerna.engine.api.IModelEngine} API. + */ +public final class ModelEngineNodeExecutor implements IAutomationNodeExecutor { + + private static final Logger classLogger = LogManager.getLogger(ModelEngineNodeExecutor.class); + + @Override + public Object execute(AutomationNodeContext ctx) throws Exception { + Map config = ctx.config(); + String nodeLabel = ctx.nodeLabel(); + Map scope = ctx.scope(); + Map configMap = ctx.configMap(); + + String engineId = NodeConfigHelper.required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); + String operation = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_LLM); + String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); + + resolvedEngineId = SecurityQueryUtils.testUserEngineIdForAlias(ctx.insight().getUser(), resolvedEngineId); + if (!SecurityEngineUtils.userCanViewEngine(ctx.insight().getUser(), resolvedEngineId)) { + throw new IllegalArgumentException( + "Model-engine node \"" + nodeLabel + "\": engine does not exist or user does not have access: " + resolvedEngineId); + } + + IModelEngine engine = Utility.getModel(resolvedEngineId); + if (engine == null) { + throw new IllegalArgumentException("Model-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); + } + + classLogger.debug("Model-engine node \"{}\" executing operation={} via engine {}", nodeLabel, operation, resolvedEngineId); + switch (operation) { + case AutomationConstants.OP_EMBEDDINGS: { + String values = NodeConfigHelper.required(config, AutomationConstants.CONFIG_VALUES, nodeLabel); + String resolvedValues = AutomationExecutionUtils.resolve(values, scope, configMap); + List valueList = Arrays.stream(resolvedValues.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + EmbeddingsModelEngineResponse response = engine.embeddings(valueList, ctx.insight(), null); + return response.getResponse(); + } + default: { + // llm (and vision/ner as fallback — both use ask() with the primary command field) + String command = NodeConfigHelper.required(config, AutomationConstants.CONFIG_COMMAND, nodeLabel); + String resolvedCommand = AutomationExecutionUtils.resolve(command, scope, configMap); + String context = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_CONTEXT); + String resolvedContext = (context != null) + ? AutomationExecutionUtils.resolve(context, scope, configMap) : null; + Map params = parseParams(config, scope, configMap, nodeLabel); + @SuppressWarnings("rawtypes") + AskModelEngineResponse response = engine.ask(resolvedCommand, resolvedContext, ctx.insight(), params); + return response.getResponse(); + } + } + } + + @SuppressWarnings("unchecked") + private static Map parseParams(Map config, + Map scope, Map configMap, String nodeLabel) { + String paramValues = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_PARAM_VALUES); + if (paramValues == null) return null; + String resolved = AutomationExecutionUtils.resolve(paramValues, scope, configMap); + try { + return AutomationExecutionUtils.GSON.fromJson(resolved, AutomationExecutionUtils.MAP_TYPE); + } catch (Exception e) { + throw new IllegalArgumentException("Model-engine node \"" + nodeLabel + "\": paramValues is not valid JSON: " + e.getMessage(), e); + } + } + +} diff --git a/src/prerna/reactor/automation/nodes/NodeConfigHelper.java b/src/prerna/reactor/automation/nodes/NodeConfigHelper.java new file mode 100644 index 00000000000..265ac432235 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/NodeConfigHelper.java @@ -0,0 +1,71 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; + +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +/** + * Delegates to {@link AutomationExecutionUtils} for config-extraction helpers. + * + *

The canonical implementations now live in {@code AutomationExecutionUtils}. This class + * is kept as a thin delegation shim so existing callers in the {@code nodes} sub-package + * continue to compile without modification. It will be removed in a later cleanup pass once + * those callers are updated to reference {@code AutomationExecutionUtils} directly. + * + * @deprecated Use {@link AutomationExecutionUtils#required}, {@link AutomationExecutionUtils#optional}, + * and {@link AutomationExecutionUtils#optionalInt} directly. + */ +@Deprecated +final class NodeConfigHelper { + + private NodeConfigHelper() { + // utility class + } + + /** @see AutomationExecutionUtils#required(Map, String, String) */ + static String required(Map config, String key, String nodeLabel) { + return AutomationExecutionUtils.required(config, key, nodeLabel); + } + + /** @see AutomationExecutionUtils#optional(Map, String, String) */ + static String optional(Map config, String key, String def) { + return AutomationExecutionUtils.optional(config, key, def); + } + + /** @see AutomationExecutionUtils#optional(Map, String) */ + static String optional(Map config, String key) { + return AutomationExecutionUtils.optional(config, key); + } + + /** @see AutomationExecutionUtils#optionalInt(Map, String, int) */ + static int optionalInt(Map config, String key, int def) { + return AutomationExecutionUtils.optionalInt(config, key, def); + } +} diff --git a/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java new file mode 100644 index 00000000000..ed655ff4290 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/StorageEngineNodeExecutor.java @@ -0,0 +1,192 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Base64; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityQueryUtils; +import prerna.engine.api.IStorageEngine; +import prerna.reactor.automation.AutomationConstants; +import prerna.reactor.automation.utils.AutomationExecutionUtils; +import prerna.util.AssetUtility; +import prerna.util.Utility; + +/** + * Executes a storage-engine automation node. Supports file transfer ({@code download}, + * {@code upload}), deletion ({@code delete}), in-memory base64 reads ({@code read-base64}), + * and directory listing ({@code list}) via the platform {@link prerna.engine.api.IStorageEngine} API. + * + *

Config fields (from {@code node.config}): + *

    + *
  • {@code engineId} (required) — UUID or alias of the target storage engine
    • + *
  • {@code operation} (optional) — one of {@code list} (default), {@code download}, + * {@code upload}, {@code delete}, {@code read-base64}
    • + *
  • {@code storagePath} (required for most operations) — path within the storage engine
    • + *
  • {@code filePath} (required for download/upload) — relative path under the owning + * project's automation runtime directory
    • + *
+ */ +public final class StorageEngineNodeExecutor implements IAutomationNodeExecutor { + + private static final Logger classLogger = LogManager.getLogger(StorageEngineNodeExecutor.class); + private static final String AUTOMATION_RUNTIME_FILES_FOLDER = "automation-files"; + + @Override + public Object execute(AutomationNodeContext ctx) throws Exception { + Map config = ctx.config(); + String nodeLabel = ctx.nodeLabel(); + Map scope = ctx.scope(); + Map configMap = ctx.configMap(); + + String engineId = NodeConfigHelper.required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); + String operation = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_LIST); + String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); + + resolvedEngineId = SecurityQueryUtils.testUserEngineIdForAlias(ctx.insight().getUser(), resolvedEngineId); + boolean mutating = AutomationConstants.OP_UPLOAD.equals(operation) + || AutomationConstants.OP_DELETE.equals(operation); + boolean authorized = mutating + ? SecurityEngineUtils.userCanEditEngine(ctx.insight().getUser(), resolvedEngineId) + : SecurityEngineUtils.userCanViewEngine(ctx.insight().getUser(), resolvedEngineId); + if (!authorized) { + throw new IllegalArgumentException( + "Storage-engine node \"" + nodeLabel + "\": engine does not exist or user does not have access: " + resolvedEngineId); + } + + IStorageEngine engine = Utility.getStorage(resolvedEngineId); + if (engine == null) { + throw new IllegalArgumentException("Storage-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); + } + + classLogger.debug("Storage-engine node \"{}\" executing operation={} via engine {}", nodeLabel, operation, resolvedEngineId); + switch (operation) { + case AutomationConstants.OP_DOWNLOAD: { + String storagePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); + String filePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_FILE_PATH, nodeLabel); + String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); + String resolvedFile = resolveLocalPath(ctx.projectId(), + AutomationExecutionUtils.resolve(filePath, scope, configMap)); + engine.copyToLocal(resolvedStorage, resolvedFile); + return "Downloaded: " + resolvedStorage; + } + case AutomationConstants.OP_UPLOAD: { + String storagePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); + String filePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_FILE_PATH, nodeLabel); + String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); + String resolvedFile = resolveLocalPath(ctx.projectId(), + AutomationExecutionUtils.resolve(filePath, scope, configMap)); + engine.copyToStorage(resolvedFile, resolvedStorage, null); + return "Uploaded: " + filePath; + } + case AutomationConstants.OP_DELETE: { + String storagePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); + String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); + engine.deleteFromStorage(resolvedStorage); + return "Deleted: " + resolvedStorage; + } + case AutomationConstants.OP_READ_BASE64: { + String storagePath = NodeConfigHelper.required(config, AutomationConstants.CONFIG_STORAGE_PATH, nodeLabel); + String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); + byte[] bytes = engine.readBlobToMemory(resolvedStorage); + return Base64.getEncoder().encodeToString(bytes); + } + default: { + // list + String storagePath = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_STORAGE_PATH, AutomationConstants.DEFAULT_STORAGE_PATH); + String resolvedStorage = AutomationExecutionUtils.resolve(storagePath, scope, configMap); + List files = engine.list(resolvedStorage); + return files; + } + } + } + + /** + * Resolves a configured local path inside a project-owned runtime directory. + * + *

Automation definitions must never grant arbitrary local server file-system access. Both + * upload and download operations are therefore confined to + * {@code /automation-files}. Existing symbolic links are rejected so a path + * within that directory cannot escape it. + * + * @param projectId owning automation project + * @param configuredPath relative user-configured path + * @return normalized absolute path inside the project runtime directory + * @throws IOException if the runtime directory cannot be created + */ + private static String resolveLocalPath(String projectId, String configuredPath) throws IOException { + if (configuredPath == null || configuredPath.isBlank()) { + throw new IllegalArgumentException("Storage-engine node local path must not be empty"); + } + if (configuredPath.matches("^[A-Za-z]:[\\\\/].*")) { + throw new IllegalArgumentException("Storage-engine node local path must be relative to the automation workspace"); + } + + Path configured = Paths.get(configuredPath); + if (configured.isAbsolute()) { + throw new IllegalArgumentException("Storage-engine node local path must be relative to the automation workspace"); + } + + Path workspace = Paths.get(AssetUtility.getProjectAppRootFolder(projectId), + AUTOMATION_RUNTIME_FILES_FOLDER).toAbsolutePath().normalize(); + Files.createDirectories(workspace); + + Path resolved = workspace.resolve(configured).normalize(); + if (!resolved.startsWith(workspace)) { + throw new IllegalArgumentException("Storage-engine node local path cannot leave the automation workspace"); + } + rejectSymbolicLinks(workspace, resolved); + return resolved.toString(); + } + + /** + * Rejects existing symbolic links along a path before it is handed to a storage engine. + * + * @param workspace project-owned automation runtime directory + * @param resolved normalized path inside the workspace + */ + private static void rejectSymbolicLinks(Path workspace, Path resolved) { + Path current = workspace; + for (Path segment : workspace.relativize(resolved)) { + current = current.resolve(segment); + if (Files.exists(current) && Files.isSymbolicLink(current)) { + throw new IllegalArgumentException("Storage-engine node local path cannot contain symbolic links"); + } + } + } + +} diff --git a/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java b/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java new file mode 100644 index 00000000000..92ac2d21dfe --- /dev/null +++ b/src/prerna/reactor/automation/nodes/VectorEngineNodeExecutor.java @@ -0,0 +1,131 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityQueryUtils; +import prerna.engine.api.IVectorDatabaseEngine; +import prerna.reactor.automation.AutomationConstants; +import prerna.reactor.automation.utils.AutomationExecutionUtils; +import prerna.util.Utility; + +/** + * Executes a vector-engine automation node. Supports document ingestion ({@code add-file}, + * {@code add-csv}), deletion ({@code delete}), listing ({@code list}), and semantic search + * ({@code search}) via the platform {@link prerna.engine.api.IVectorDatabaseEngine} API. + * + *

Config fields (from {@code node.config}): + *

    + *
  • {@code engineId} (required) — UUID or alias of the target vector engine
    • + *
  • {@code operation} (optional) — one of {@code search} (default), {@code add-file}, + * {@code add-csv}, {@code delete}, {@code list}
    • + *
  • {@code filePath} (required for add-file/add-csv) — comma-separated file paths
    • + *
  • {@code fileNames} (required for delete) — comma-separated document names to remove
    • + *
  • {@code command} (required for search) — the query string
    • + *
  • {@code limit} (optional for search) — max results; default {@value AutomationConstants#DEFAULT_VECTOR_SEARCH_LIMIT}
    • + *
+ */ +public final class VectorEngineNodeExecutor implements IAutomationNodeExecutor { + + private static final Logger classLogger = LogManager.getLogger(VectorEngineNodeExecutor.class); + + @Override + public Object execute(AutomationNodeContext ctx) throws Exception { + Map config = ctx.config(); + String nodeLabel = ctx.nodeLabel(); + Map scope = ctx.scope(); + Map configMap = ctx.configMap(); + + String engineId = NodeConfigHelper.required(config, AutomationConstants.CONFIG_ENGINE_ID, nodeLabel); + String operation = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_OPERATION, AutomationConstants.OP_SEARCH); + String resolvedEngineId = AutomationExecutionUtils.resolve(engineId, scope, configMap); + + resolvedEngineId = SecurityQueryUtils.testUserEngineIdForAlias(ctx.insight().getUser(), resolvedEngineId); + boolean mutating = AutomationConstants.OP_ADD_FILE.equals(operation) + || AutomationConstants.OP_ADD_CSV.equals(operation) + || AutomationConstants.OP_DELETE.equals(operation); + boolean authorized = mutating + ? SecurityEngineUtils.userCanEditEngine(ctx.insight().getUser(), resolvedEngineId) + : SecurityEngineUtils.userCanViewEngine(ctx.insight().getUser(), resolvedEngineId); + if (!authorized) { + throw new IllegalArgumentException( + "Vector-engine node \"" + nodeLabel + "\": engine does not exist or user does not have access: " + resolvedEngineId); + } + + IVectorDatabaseEngine engine = Utility.getVectorDatabase(resolvedEngineId); + if (engine == null) { + throw new IllegalArgumentException("Vector-engine node \"" + nodeLabel + "\": engine not found: " + resolvedEngineId); + } + + classLogger.debug("Vector-engine node \"{}\" executing operation={} via engine {}", nodeLabel, operation, resolvedEngineId); + switch (operation) { + case AutomationConstants.OP_ADD_FILE: + case AutomationConstants.OP_ADD_CSV: { + String filePaths = NodeConfigHelper.required(config, AutomationConstants.CONFIG_FILE_PATH, nodeLabel); + String resolvedPaths = AutomationExecutionUtils.resolve(filePaths, scope, configMap); + List paths = Arrays.stream(resolvedPaths.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + engine.addDocument(paths, null); + return "Added " + paths.size() + " file(s)"; + } + case AutomationConstants.OP_LIST: { + List> docs = engine.listDocuments(null); + return docs; + } + case AutomationConstants.OP_DELETE: { + String fileNames = NodeConfigHelper.required(config, AutomationConstants.CONFIG_FILE_NAMES, nodeLabel); + String resolvedNames = AutomationExecutionUtils.resolve(fileNames, scope, configMap); + List names = Arrays.stream(resolvedNames.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + engine.removeDocument(names, null); + return "Deleted " + names.size() + " file(s)"; + } + default: { + // search + String command = NodeConfigHelper.required(config, AutomationConstants.CONFIG_COMMAND, nodeLabel); + String resolvedCommand = AutomationExecutionUtils.resolve(command, scope, configMap); + int limit = NodeConfigHelper.optionalInt(config, AutomationConstants.CONFIG_LIMIT, AutomationConstants.DEFAULT_VECTOR_SEARCH_LIMIT); + List> results = engine.nearestNeighbor(ctx.insight(), resolvedCommand, limit, null); + return results; + } + } + } + +} diff --git a/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java new file mode 100644 index 00000000000..320937e4305 --- /dev/null +++ b/src/prerna/reactor/automation/nodes/WaitNodeExecutor.java @@ -0,0 +1,96 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.nodes; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.reactor.automation.AutomationCancelledException; +import prerna.reactor.automation.AutomationConstants; +import prerna.reactor.automation.AutomationDatabaseUtility; +import prerna.reactor.automation.utils.AutomationExecutionUtils; + +/** + * Executes a "wait" node: sleeps for the configured number of seconds. + * The {@code seconds} value supports {@code ${var}} template substitution. + * Maximum {@value AutomationConstants#WAIT_MAX_SECONDS} seconds (1 hour) per invocation. + * + *

Sleeps in {@value AutomationConstants#WAIT_CANCEL_CHECK_INTERVAL_SECONDS}-second chunks, + * checking the cancellation flag between each chunk. This fixes the bug where a single + * {@code Thread.sleep(N)} call would block for the full duration even after a cancel request + * was received - the cancel check was only between nodes, so a long wait could not be + * interrupted until it completed naturally. + */ +public final class WaitNodeExecutor implements IAutomationNodeExecutor { + + private static final Logger classLogger = LogManager.getLogger(WaitNodeExecutor.class); + + @Override + public Object execute(AutomationNodeContext ctx) throws Exception { + Map config = ctx.config(); + String nodeLabel = ctx.nodeLabel(); + + String secondsTemplate = NodeConfigHelper.optional(config, AutomationConstants.CONFIG_SECONDS, + String.valueOf(AutomationConstants.WAIT_DEFAULT_SECONDS)); + String resolved = AutomationExecutionUtils.resolve(secondsTemplate, ctx.scope(), ctx.configMap()); + + int seconds; + try { + seconds = Integer.parseInt(resolved.trim()); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Wait node \"" + nodeLabel + + "\" - seconds value is not a valid integer after resolution: \"" + resolved + "\""); + } + seconds = Math.min(Math.max(seconds, AutomationConstants.WAIT_MIN_SECONDS), AutomationConstants.WAIT_MAX_SECONDS); + classLogger.debug("Wait node \"{}\" sleeping {} seconds", nodeLabel, seconds); + + // Sleep in chunks so cancel requests are honored mid-wait rather than waiting + // for the full duration to complete. + int remaining = seconds; + while (remaining > 0) { + // isCancelRequested checks the DB; this is intentionally bounded by + // WAIT_CANCEL_CHECK_INTERVAL_SECONDS to avoid thrashing the JDBC pool. + if (ctx.cancelFlag().get() || AutomationDatabaseUtility.isCancelRequested(ctx.runId())) { + throw new AutomationCancelledException("Wait node \"" + nodeLabel + "\" cancelled"); + } + int chunk = Math.min(remaining, AutomationConstants.WAIT_CANCEL_CHECK_INTERVAL_SECONDS); + try { + TimeUnit.SECONDS.sleep(chunk); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Wait node \"" + nodeLabel + "\" was interrupted"); + } + remaining -= chunk; + } + + return seconds + " seconds"; + } +} diff --git a/src/prerna/reactor/automation/utils/AutomationExecutionUtils.java b/src/prerna/reactor/automation/utils/AutomationExecutionUtils.java new file mode 100644 index 00000000000..a970cd67586 --- /dev/null +++ b/src/prerna/reactor/automation/utils/AutomationExecutionUtils.java @@ -0,0 +1,630 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.utils; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Type; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.text.StringSubstitutor; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonPrimitive; +import com.google.gson.JsonSerializer; +import com.google.gson.reflect.TypeToken; + +import prerna.auth.User; +import prerna.util.AssetUtility; +import prerna.reactor.automation.AutomationConstants; + +/** + * Shared static utilities for the automation execution engine. + * + *

Centralizes logic shared across {@link prerna.reactor.automation.TriggerAutomationReactor} and + * {@link prerna.reactor.automation.RunAutomationNodeReactor}. + */ +public final class AutomationExecutionUtils { + + private static final Logger classLogger = LogManager.getLogger(AutomationExecutionUtils.class); + + /** Prefix for config-map lookups within a template, e.g. {@code ${config.API_KEY}}. */ + private static final String CONFIG_VAR_PREFIX = "config."; + + /** Reusable {@link TypeToken} type for {@code Map} deserialization. */ + public static final Type MAP_TYPE = new TypeToken>() {}.getType(); + + /** Reusable {@link TypeToken} type for {@code List>} deserialization. */ + private static final Type LIST_OBJ_MAP_TYPE = new TypeToken>>() {}.getType(); + + /** + * Shared Gson instance for the whole automation engine - public so the + * {@code nodes} sub-package has one shared instance to reuse instead of each + * file declaring its own. + */ + public static final Gson GSON = new GsonBuilder() + .disableHtmlEscaping() + .registerTypeHierarchyAdapter(ZoneId.class, + (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.getId())) + .registerTypeAdapter(ZonedDateTime.class, + (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.toString())) + .registerTypeAdapter(OffsetDateTime.class, + (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.toString())) + .registerTypeAdapter(LocalDateTime.class, + (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.toString())) + .registerTypeAdapter(LocalDate.class, + (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.toString())) + .registerTypeAdapter(Instant.class, + (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.toString())) + .registerTypeHierarchyAdapter(Throwable.class, + (JsonSerializer) (src, t, ctx) -> new JsonPrimitive(src.toString())) + .create(); + + private AutomationExecutionUtils() {} + + /** + * Resolves {@code ${varName}} and {@code ${config.KEY}} placeholders in a template string + * using {@link StringSubstitutor} (the platform's standard {@code ${...}} templating helper, + * also used by {@code AbstractPythonModelEngine.fillVars}). Recursive substitution is + * disabled so a resolved value is never itself re-scanned for placeholders, and unresolved + * placeholders are left untouched rather than throwing. + */ + public static String resolve(String template, Map scope, Map configMap) { + if (template == null) return ""; + + Map vars = new HashMap<>(); + for (Map.Entry e : configMap.entrySet()) { + vars.put(CONFIG_VAR_PREFIX + e.getKey(), e.getValue()); + } + for (Map.Entry e : scope.entrySet()) { + if (e.getValue() != null) { + vars.put(e.getKey(), e.getValue()); + } + } + + StringSubstitutor sub = new StringSubstitutor(vars); + sub.setEnableUndefinedVariableException(false); + sub.setDisableSubstitutionInValues(true); + return sub.replace(template); + } + + /** + * Returns the per-node timeout from the node definition, defaulting to + * {@link AutomationConstants#DEFAULT_TIMEOUT_SECONDS}. + */ + public static int getNodeTimeout(Map node) { + Object timeout = node.get(AutomationConstants.CONFIG_TIMEOUT_SECONDS); + if (timeout instanceof Number) { + return ((Number) timeout).intValue(); + } + return AutomationConstants.DEFAULT_TIMEOUT_SECONDS; + } + + /** + * Loads {@code automation-config.json} for a project and returns key->value pairs. + * Returns an empty map if the file does not exist or cannot be parsed. + */ + @SuppressWarnings("unchecked") + public static Map loadConfig(String projectId) { + Map map = new HashMap<>(); + try { + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File f = Paths.get(portalsFolder, AutomationConstants.AUTOMATION_CONFIG_FILE_NAME).toFile(); + if (!f.exists()) return map; + String json = Files.readString(f.toPath(), StandardCharsets.UTF_8); + List> entries = GSON.fromJson(json, LIST_OBJ_MAP_TYPE); + if (entries != null) { + for (Map entry : entries) { + String key = (String) entry.get(AutomationConstants.CONFIG_ENTRY_KEY); + String value = (String) entry.get(AutomationConstants.CONFIG_ENTRY_VALUE); + if (key != null && value != null) map.put(key, value); + } + } + } catch (Exception e) { + classLogger.warn("Failed to load automation config for project {}", projectId, e); + } + return map; + } + + /** + * Applies an output transform to a raw pixel result. + * + *

Supported modes: {@code rows-as-objects}, {@code first-row}, {@code column}, + * {@code jsonpath}. Returns raw serialized JSON when config is null or mode is {@code raw}. + */ + @SuppressWarnings("unchecked") + public static String applyOutputTransform(Object rawResult, Map transformConfig) { + String rawStr = serializeRaw(rawResult); + if (transformConfig == null) return rawStr; + + String mode = (String) transformConfig.getOrDefault(AutomationConstants.TRANSFORM_MODE, AutomationConstants.TRANSFORM_MODE_RAW); + switch (mode) { + case AutomationConstants.TRANSFORM_MODE_ROWS_AS_OBJECTS: return transformRowsAsObjects(rawStr); + case AutomationConstants.TRANSFORM_MODE_FIRST_ROW: return transformFirstRow(rawStr); + case AutomationConstants.TRANSFORM_MODE_COLUMN: return transformColumn(rawStr, (String) transformConfig.get(AutomationConstants.TRANSFORM_COLUMN)); + case AutomationConstants.TRANSFORM_MODE_JSONPATH: return transformJsonPath(rawStr, (String) transformConfig.get(AutomationConstants.TRANSFORM_PATH)); + default: return rawStr; + } + } + + /** Serializes a raw pixel result to a JSON string. */ + public static String serializeRaw(Object rawResult) { + if (rawResult == null) return ""; + if (rawResult instanceof String) return (String) rawResult; + return GSON.toJson(rawResult); + } + + // -- Private transform helpers ------------------------------------------------- + + @SuppressWarnings("unchecked") + private static String transformRowsAsObjects(String rawStr) { + Map data = extractDataset(parseJsonAny(rawStr)); + if (data == null) return rawStr; + List headers = (List) data.get(AutomationConstants.DATASET_HEADERS); + List> rows = (List>) data.get(AutomationConstants.DATASET_VALUES); + if (headers == null || rows == null) return rawStr; + List> result = new ArrayList<>(); + for (List row : rows) { + Map rowMap = new HashMap<>(); + for (int i = 0; i < headers.size() && i < row.size(); i++) { + rowMap.put(headers.get(i), row.get(i)); + } + result.add(rowMap); + } + return GSON.toJson(result); + } + + @SuppressWarnings("unchecked") + private static String transformFirstRow(String rawStr) { + Map data = extractDataset(parseJsonAny(rawStr)); + if (data == null) return rawStr; + List headers = (List) data.get(AutomationConstants.DATASET_HEADERS); + List> rows = (List>) data.get(AutomationConstants.DATASET_VALUES); + if (headers == null || rows == null || rows.isEmpty()) return rawStr; + Map rowMap = new HashMap<>(); + List first = rows.get(0); + for (int i = 0; i < headers.size() && i < first.size(); i++) { + rowMap.put(headers.get(i), first.get(i)); + } + return GSON.toJson(rowMap); + } + + @SuppressWarnings("unchecked") + private static String transformColumn(String rawStr, String colName) { + if (colName == null || colName.isEmpty()) return rawStr; + Map data = extractDataset(parseJsonAny(rawStr)); + if (data == null) return rawStr; + List headers = (List) data.get(AutomationConstants.DATASET_HEADERS); + List> rows = (List>) data.get(AutomationConstants.DATASET_VALUES); + if (headers == null || rows == null) return rawStr; + int colIdx = headers.indexOf(colName); + if (colIdx < 0) return rawStr; + List col = new ArrayList<>(); + for (List row : rows) col.add(colIdx < row.size() ? row.get(colIdx) : null); + return GSON.toJson(col); + } + + @SuppressWarnings("unchecked") + private static String transformJsonPath(String rawStr, String path) { + if (path == null || path.isEmpty()) return rawStr; + try { + Object current = parseJsonAny(rawStr); + for (String segment : path.split("\\.")) { + if (!(current instanceof Map)) break; + current = ((Map) current).get(segment); + } + if (current == null) return ""; + return current instanceof String ? (String) current : GSON.toJson(current); + } catch (Exception e) { + return rawStr; + } + } + + /** + * Normalises any of the three dataset formats into a canonical {@code {headers, values}} map: + * 1. List<Map> (rows-as-objects) - produced by {@link DatabaseEngineNodeExecutor} + * 2. {@code {data: {headers, values}}} - SEMOSS wrapped envelope + * 3. {@code {headers, values}} - SEMOSS direct + */ + @SuppressWarnings("unchecked") + private static Map extractDataset(Object parsed) { + if (parsed == null) return null; + + // Format 1: rows-as-objects list from DatabaseEngineNodeExecutor + if (parsed instanceof List) { + List list = (List) parsed; + if (list.isEmpty()) return null; + Object first = list.get(0); + if (!(first instanceof Map)) return null; + List headers = new ArrayList<>(((Map) first).keySet()); + List> values = new ArrayList<>(); + for (Object item : list) { + if (item instanceof Map) { + Map row = (Map) item; + List rowVals = new ArrayList<>(); + for (String h : headers) rowVals.add(row.get(h)); + values.add(rowVals); + } + } + Map result = new HashMap<>(); + result.put(AutomationConstants.DATASET_HEADERS, headers); + result.put(AutomationConstants.DATASET_VALUES, values); + return result; + } + + if (!(parsed instanceof Map)) return null; + Map map = (Map) parsed; + + // Format 2: {data: {headers, values}} + if (map.containsKey(AutomationConstants.DATASET_DATA) && map.get(AutomationConstants.DATASET_DATA) instanceof Map) { + return (Map) map.get(AutomationConstants.DATASET_DATA); + } + // Format 3: {headers, values} + if (map.containsKey(AutomationConstants.DATASET_HEADERS) && map.containsKey(AutomationConstants.DATASET_VALUES)) return map; + return null; + } + + /** Parses JSON to Object - returns List for arrays, Map for objects (handles all executor output shapes). */ + private static Object parseJsonAny(String json) { + if (json == null || json.isBlank()) return null; + try { + return GSON.fromJson(json, Object.class); + } catch (Exception e) { + return null; + } + } + + private static Map parseJson(String json) { + if (json == null || json.isBlank()) return null; + try { + return GSON.fromJson(json, MAP_TYPE); + } catch (Exception e) { + return null; + } + } + + // -- Scope building ------------------------------------------------------------ + + /** + * Builds the initial variable scope for an automation run, seeded with {@code date}, + * {@code triggered_at}, and {@code run_id} (when non-blank). + * + * @param runId the run ID to seed into scope, or {@code null} for test runs + * @param user the triggering user - used to localise {@code date} and {@code triggered_at} + * to the user's configured timezone; falls back to UTC when {@code null} or + * when no zone has been set on the user + */ + public static Map buildInitialScope(String runId, User user) { + Map scope = new HashMap<>(); + ZoneId zone = (user != null && user.getZoneId() != null) ? user.getZoneId() : ZoneId.of("UTC"); + ZonedDateTime now = ZonedDateTime.now(zone); + scope.put(AutomationConstants.SCOPE_DATE, now.format(DateTimeFormatter.ISO_LOCAL_DATE)); + scope.put(AutomationConstants.SCOPE_TRIGGERED_AT, now.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + if (runId != null && !runId.isBlank()) scope.put(AutomationConstants.SCOPE_RUN_ID, runId); + return scope; + } + + /** + * Builds the human-readable, per-workflow summary surfaced to MCP/agent consumers as the + * primary tool result (instead of a raw run-detail JSON blob). + * + *

Resolves {@code automation.json}'s optional {@link AutomationConstants#DOC_RESULT_MESSAGE_TEMPLATE} + * (e.g. {@code "Indexed ${file_count} files"}) against the final run scope + config using the + * same {@code ${var}}/{@code ${config.KEY}} substitution as node templates. Falls back to a + * generic completed-nodes message when the template is absent, blank, or resolves to blank. + * + * @param doc the parsed {@code automation.json} document + * @param finalScope the run's scope after all nodes completed (output vars included) + * @param configMap project automation config key-value pairs + * @param completedCount number of nodes that completed successfully + * @param totalCount total number of nodes in the run + */ + public static String buildSummaryMessage(Map doc, Map finalScope, + Map configMap, int completedCount, int totalCount) { + Object rawTemplate = doc != null ? doc.get(AutomationConstants.DOC_RESULT_MESSAGE_TEMPLATE) : null; + String template = strCfg(rawTemplate); + if (template != null) { + String resolved = resolve(template, finalScope, configMap); + if (resolved != null && !resolved.isBlank()) { + return resolved; + } + } + return "Automation completed successfully (" + completedCount + "/" + totalCount + " nodes)."; + } + + /** + * Builds the user-facing portion of the LLM prompt for {@code GenerateRunSummaryReactor}. + * Describes the overall run status and each step's label, status, and output preview. + * + * @param runDetail map returned by {@link AutomationDatabaseUtility#getRunDetail} + * @param nodeOutputs ordered list returned by {@link AutomationDatabaseUtility#getNodeOutputsForRun} + * @return a plain-text prompt fragment suitable for appending after the system prompt + */ + public static String buildRunSummaryPrompt(Map runDetail, + List> nodeOutputs) { + StringBuilder sb = new StringBuilder(); + + String status = strCfg(runDetail.get(AutomationConstants.STATUS)); + sb.append("Run status: ").append(status != null ? status : "unknown").append("\n"); + + String failedNodeId = strCfg(runDetail.get(AutomationConstants.FAILED_NODE_ID)); + String errorMsg = strCfg(runDetail.get(AutomationConstants.ERROR_MESSAGE)); + if (failedNodeId != null) { + sb.append("Failed at: ").append(failedNodeId).append("\n"); + } + if (errorMsg != null) { + sb.append("Error: ").append(errorMsg).append("\n"); + } + + if (!nodeOutputs.isEmpty()) { + sb.append("\nSteps:\n"); + for (Map node : nodeOutputs) { + String label = strCfg(node.get(AutomationConstants.NODE_LABEL)); + String nodeStatus = strCfg(node.get(AutomationConstants.STATUS)); + String preview = strCfg(node.get(AutomationConstants.OUTPUT_PREVIEW)); + String errDetail = strCfg(node.get(AutomationConstants.ERROR_MESSAGE)); + + sb.append("- ") + .append(label != null ? label : AutomationConstants.UNNAMED_NODE_LABEL) + .append(" [").append(nodeStatus != null ? nodeStatus : "unknown").append("]"); + + if (preview != null) { + String truncated = preview.length() > AutomationConstants.SUMMARY_PROMPT_PREVIEW_MAX_LENGTH + ? preview.substring(0, AutomationConstants.SUMMARY_PROMPT_PREVIEW_MAX_LENGTH) + "..." + : preview; + sb.append(": ").append(truncated); + } else if (errDetail != null) { + sb.append(": ").append(errDetail); + } + sb.append("\n"); + } + } + + return sb.toString(); + } + + /** Truncates a string to {@link AutomationConstants#OUTPUT_PREVIEW_MAX_LENGTH} chars. */ + public static String generatePreview(String s) { + if (s == null) return null; + return s.length() <= AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH + ? s : s.substring(0, AutomationConstants.OUTPUT_PREVIEW_MAX_LENGTH); + } + + // -- Playground input helpers -------------------------------------------------- + + /** + * Extracts playground input values from the Pixel MAP noun and overwrites the matching + * config field on each node before execution. Nodes without a {@code playgroundFillable} + * list or whose listed fields are absent from the node config are skipped with a warning. + * + * @param nodes ordered node list from the automation document (mutated in place) + * @param inputsMap raw MAP noun value from the {@code inputs} Pixel parameter, or {@code null} + * @return the flattened {@code String → String} inputs map (empty when no inputs were passed) + */ + @SuppressWarnings("unchecked") + public static Map applyPlaygroundInputs(List> nodes, Map inputsMap) { + Map playgroundInputs = new HashMap<>(); + if (inputsMap != null) { + for (Map.Entry entry : inputsMap.entrySet()) { + if (entry.getValue() != null) { + playgroundInputs.put(entry.getKey(), entry.getValue().toString()); + } + } + } + if (!playgroundInputs.isEmpty()) { + for (Map node : nodes) { + List fillable = (List) node.get("playgroundFillable"); + if (fillable == null || fillable.isEmpty()) continue; + String nodeLabel = (String) node.get(AutomationConstants.NODE_FIELD_LABEL); + Map config = (Map) node.get(AutomationConstants.NODE_FIELD_CONFIG); + if (config == null) continue; + for (String fieldName : fillable) { + String paramName = buildPlaygroundParamName(nodeLabel, fieldName); + String value = playgroundInputs.get(paramName); + if (value != null) { + if (config.containsKey(fieldName)) { + config.put(fieldName, value); + } else { + classLogger.warn("Playground input '{}' targets field '{}' which does not exist in node '{}' config - skipping", + paramName, fieldName, nodeLabel); + } + } + } + } + } + return playgroundInputs; + } + + /** + * Builds a stable MCP parameter name for a playground-fillable node field. + * Slugifies the node label (lowercase, non-alphanumeric chars -> underscore, collapsed) + * and appends the field name. Falls back to {@code "input_" + fieldName} if label is blank. + * + * @param nodeLabel the node's display label (may be null or empty) + * @param fieldName the config field name (e.g. "expression", "command") + * @return a deterministic, URL-safe parameter name + */ + public static String buildPlaygroundParamName(String nodeLabel, String fieldName) { + if (nodeLabel != null && !nodeLabel.trim().isEmpty()) { + String slug = nodeLabel.trim() + .toLowerCase() + .replaceAll("[^a-z0-9]+", "_") + .replaceAll("^_+|_+$", ""); + if (!slug.isEmpty()) { + return slug + "_" + fieldName; + } + } + return "input_" + fieldName; + } + + // -- Config value coercion ----------------------------------------------------- + + /** + * Returns the trimmed string form of a node config value, or {@code null} if the value is + * {@code null} or blank. + */ + public static String strCfg(Object v) { + return (v != null && !v.toString().isBlank()) ? v.toString() : null; + } + + /** + * Normalizes a config value that may arrive as either an already-parsed {@code Map} or a + * raw JSON string (e.g. {@code inputMapping}, {@code paramValues}). Returns an empty map + * if absent or unparseable. + */ + @SuppressWarnings("unchecked") + public static Map coerceToMap(Object raw) { + if (raw instanceof Map) { + return (Map) raw; + } + if (raw instanceof String str && !str.isBlank()) { + Map parsed = parseJson(str); + if (parsed != null) return parsed; + } + return new HashMap<>(); + } + + // -- Node config access helpers ----------------------------------------------- + // These mirror the methods in NodeConfigHelper (nodes sub-package), which now + // delegates to these. New callers should use AutomationExecutionUtils directly. + + /** + * Returns the string value of {@code key} from {@code config}, throwing if absent or blank. + * + * @param config node config map + * @param key config key to look up + * @param nodeLabel display label of the containing node (used in the error message) + */ + public static String required(Map config, String key, String nodeLabel) { + Object v = config.get(key); + if (v == null || v.toString().isBlank()) { + throw new IllegalArgumentException( + "Node \"" + nodeLabel + "\": '" + key + "' is required"); + } + return v.toString(); + } + + /** + * Returns the string value of {@code key} from {@code config}, or {@code def} if absent or blank. + * + * @param config node config map + * @param key config key to look up + * @param def value to return when key is absent or blank + */ + public static String optional(Map config, String key, String def) { + Object v = config.get(key); + return (v == null || v.toString().isBlank()) ? def : v.toString(); + } + + /** + * Returns the string value of {@code key} from {@code config}, or {@code null} if absent or blank. + * + * @param config node config map + * @param key config key to look up + */ + public static String optional(Map config, String key) { + return optional(config, key, null); + } + + /** + * Returns the integer value of {@code key} from {@code config}, or {@code def} if absent, blank, + * or not parseable as an integer. + * + * @param config node config map + * @param key config key to look up + * @param def default value when key is absent, blank, or unparseable + */ + public static int optionalInt(Map config, String key, int def) { + Object v = config.get(key); + if (v == null) return def; + try { + return Integer.parseInt(v.toString().trim()); + } catch (NumberFormatException e) { + return def; + } + } + + // -- Automation document loading ----------------------------------------------- + + /** + * Reads a project's {@code automation.json} as a raw JSON string. + * Returns a minimal blank document string when the file does not exist or cannot be read, + * rather than throwing. Suitable for callers (e.g. LLM prompts) that need a fallback value. + * + * @param projectId the project to load the automation doc for + * @return the file contents, or {@code {"version":1,"graph":{"nodes":[],"edges":[]}}} when absent + */ + public static String loadAutomationDocOrEmpty(String projectId) { + try { + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File f = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); + if (f.exists() && f.isFile()) { + return Files.readString(f.toPath(), StandardCharsets.UTF_8); + } + } catch (IOException e) { + classLogger.warn("Could not read automation.json for project {} - returning empty doc", projectId, e); + } + return "{\"version\":1,\"graph\":{\"nodes\":[],\"edges\":[]}}"; + } + + /** + * Loads and parses a project's {@code automation.json} (graph + trigger config). + * Throws if the file is missing or unreadable. + */ + @SuppressWarnings("unchecked") + public static Map loadAutomationDoc(String projectId) { + String portalsFolder = AssetUtility.getProjectPortalsFolder(projectId); + File f = new File(portalsFolder + "/" + AutomationConstants.AUTOMATION_FILE_NAME); + if (!f.exists()) { + throw new IllegalArgumentException("No automation.json found for this project. Save an automation first."); + } + try { + String json = Files.readString(f.toPath(), StandardCharsets.UTF_8); + return GSON.fromJson(json, MAP_TYPE); + } catch (IOException e) { + throw new IllegalStateException("Failed to read automation.json: " + e.getMessage(), e); + } + } + +} diff --git a/src/prerna/reactor/automation/utils/AutomationGenerationUtils.java b/src/prerna/reactor/automation/utils/AutomationGenerationUtils.java new file mode 100644 index 00000000000..f24c0c5dc62 --- /dev/null +++ b/src/prerna/reactor/automation/utils/AutomationGenerationUtils.java @@ -0,0 +1,334 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.utils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.engine.api.IDatabaseEngine; +import prerna.engine.api.IEngine; +import prerna.engine.api.IRDBMSEngine; +import prerna.project.api.IProject; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.util.Constants; +import prerna.util.Utility; +import prerna.reactor.automation.AutomationConstants; + +/** + * LLM/generation helpers used exclusively by the AI reactor group: + * {@link prerna.reactor.automation.BuildAutomationReactor}, + * {@link prerna.reactor.automation.ExplainAutomationReactor}, {@link prerna.reactor.automation.GenerateNodeLabelReactor}, + * {@link prerna.reactor.automation.GenerateRunSummaryReactor}, {@link prerna.reactor.automation.QuickEditAutomationReactor}, + * {@link prerna.reactor.automation.AutomationAskRoomReactor}, and {@link prerna.reactor.automation.GetReactorSignatureReactor}. + * + *

Execution-time helpers (resolve, scope building, output transforms, etc.) + * live in {@link AutomationExecutionUtils}. + */ +public final class AutomationGenerationUtils { + + private static final Logger classLogger = LogManager.getLogger(AutomationGenerationUtils.class); + + /** Engine types listed in the generation prompt so the LLM can populate engineId fields. */ + static final List GENERATION_ENGINE_TYPES = Arrays.asList("DATABASE", "MODEL", "VECTOR", "STORAGE", "FUNCTION"); + + private AutomationGenerationUtils() {} + + /** + * Returns the ID of the first MODEL-type engine the user has access to, or {@code null} if none. + */ + public static String findFirstModelEngine(User user) { + try { + List> engines = SecurityEngineUtils.getUserEngineList( + user, List.of("MODEL"), null, false, null, null, null, "1", "0", null); + if (engines != null && !engines.isEmpty()) { + Object id = engines.get(0).get("database_id"); + return id != null ? String.valueOf(id) : null; + } + } catch (Exception e) { + classLogger.warn("Failed to auto-discover model engine", e); + } + return null; + } + + /** + * Extracts the text content from a model engine response map. + * Checks {@code response}, {@code output}, and {@code content} keys in order. + */ + public static String extractResponseText(Map response) { + if (response == null) return null; + Object resp = response.get("response"); + if (resp instanceof String s && !s.isBlank()) return s; + Object output = response.get("output"); + if (output instanceof String s && !s.isBlank()) return s; + Object content = response.get("content"); + if (content instanceof String s && !s.isBlank()) return s; + return null; + } + + /** + * Builds the Room options map for RunAgent - registers all user-accessible engines + * as MCP tools and includes the room's built-in tools. + */ + public static Map buildEngineMcpOptions(User user, String systemPrompt) { + List> mcpList = new ArrayList<>(); + try { + List> engines = SecurityEngineUtils.getUserEngineList( + user, GENERATION_ENGINE_TYPES, null, false, null, null, null, "50", "0", null); + if (engines != null) { + for (Map engine : engines) { + String id = engine.getOrDefault("database_id", "").toString(); + String name = engine.getOrDefault("database_name", "").toString(); + if (id.isBlank()) continue; + try { + IEngine engineObj = Utility.getEngine(id); + if (engineObj == null || !engineObj.isMCPEnabled()) continue; + } catch (Exception e) { + classLogger.debug("Skipping engine {} from MCP list - could not load: {}", id, e.getMessage()); + continue; + } + Map entry = new HashMap<>(); + entry.put("id", id); + entry.put("name", name); + entry.put("type", "ENGINE"); + mcpList.add(entry); + } + } + } catch (Exception e) { + classLogger.warn("Failed to build MCP engine list for automation room", e); + } + Map dbMakerEntry = new HashMap<>(); + dbMakerEntry.put("id", Constants.MCP_DATABASE_MAKER); + dbMakerEntry.put("name", "Database Tools"); + dbMakerEntry.put("type", "PROJECT"); + mcpList.add(dbMakerEntry); + Map roomEntry = new HashMap<>(); + roomEntry.put("id", MCPUtility.ROOM_MCP_ID); + roomEntry.put("name", MCPUtility.ROOM_MCP_NAME); + roomEntry.put("type", MCPUtility.ROOM_MCP_TYPE); + roomEntry.put("fromRoom", true); + mcpList.add(roomEntry); + Map options = new HashMap<>(); + if (systemPrompt != null && !systemPrompt.isBlank()) { + options.put("instructions", systemPrompt); + } + options.put("mcp", mcpList); + return options; + } + + /** + * Builds the "## Available engines" prompt section for generation - lists each engine the + * current user can access so the LLM can assign engineId fields correctly. + */ + public static String buildAvailableEnginesSection(User user) { + StringBuilder sb = new StringBuilder("## Available engines\n"); + try { + List> engines = SecurityEngineUtils.getUserEngineList( + user, GENERATION_ENGINE_TYPES, null, false, null, null, null, "50", "0", null); + if (engines == null || engines.isEmpty()) { + sb.append("None available - leave engineId as empty string.\n"); + return sb.toString(); + } + for (Map engine : engines) { + String id = engine.getOrDefault("database_id", "").toString(); + String name = engine.getOrDefault("database_name", "").toString(); + String type = engine.getOrDefault("engine_type", "").toString().toUpperCase(); + sb.append("- type=").append(type) + .append(" id=\"").append(id) + .append("\" name=\"").append(name).append("\""); + // For relational databases, include the table list so the model can match + // the user's intent to the right DB without needing to call get_db_schema first. + if ("DATABASE".equals(type) && !id.isBlank()) { + try { + IDatabaseEngine dbEngine = Utility.getDatabase(id); + if (dbEngine instanceof IRDBMSEngine rdbms) { + List tables = rdbms.getPixelConcepts(); + if (tables != null && !tables.isEmpty()) { + sb.append(" tables=["); + for (int i = 0; i < tables.size(); i++) { + String table = tables.get(i); + // Strip the "ConceptName__" prefix that pixel concepts sometimes include + int sep = table.lastIndexOf("__"); + sb.append(sep >= 0 ? table.substring(sep + 2) : table); + if (i < tables.size() - 1) sb.append(", "); + } + sb.append("]"); + } + } + } catch (Exception e) { + // Table list is best-effort; the model can still call get_db_schema if needed + } + } + sb.append("\n"); + } + } catch (Exception e) { + classLogger.warn("Failed to build engine list for generation prompt", e); + sb.append("(engine list unavailable - leave engineId as empty string)\n"); + } + return sb.toString(); + } + + /** + * Fetches the table/column schema (with data types) for the given list of database engine IDs. + * Formatted as a prompt section for LLM use. + */ + public static String buildSchemaForEngineIds(List engineIds) { + StringBuilder sb = new StringBuilder("## Database schema\n"); + for (String engineId : engineIds) { + try { + IDatabaseEngine dbEngine = Utility.getDatabase(engineId); + if (!(dbEngine instanceof IRDBMSEngine rdbms)) { + sb.append("Database id=\"").append(engineId).append("\": (non-relational - no table schema)\n"); + continue; + } + sb.append("Database id=\"").append(engineId).append("\":\n"); + List tables = rdbms.getPixelConcepts(); + if (tables == null || tables.isEmpty()) { + sb.append(" (no tables found)\n"); + continue; + } + for (String table : tables) { + List columns = rdbms.getPixelSelectors(table); + sb.append(" - ").append(table).append(" ("); + if (columns != null && !columns.isEmpty()) { + StringBuilder cols = new StringBuilder(); + for (String col : columns) { + int sep = col.lastIndexOf("__"); + String colName = sep >= 0 ? col.substring(sep + 2) : col; + String dataType = null; + try { + String physicalUri = rdbms.getPhysicalUriFromPixelSelector(col); + if (physicalUri != null) { + dataType = rdbms.getDataTypes(physicalUri); + } + } catch (Exception ignored) { + // Data type fetch is best-effort; column name alone is still useful + } + if (cols.length() > 0) cols.append(", "); + cols.append(colName); + if (dataType != null && !dataType.isBlank()) { + cols.append(": ").append(dataType.trim()); + } + } + sb.append(cols); + } + sb.append(")\n"); + } + } catch (Exception e) { + classLogger.warn("Failed to fetch schema for engine {}", engineId, e); + sb.append("Database id=\"").append(engineId).append("\": (schema unavailable)\n"); + } + } + return sb.toString(); + } + + /** + * Returns the list of custom reactors available in the given project, formatted as a prompt + * section for LLM use. Used to fill in app-node pixel expressions. + */ + public static String buildReactorListSection(String projectId) { + StringBuilder sb = new StringBuilder("## Available custom reactors in this project\n"); + try { + IProject project = Utility.getProject(projectId); + if (project == null) { + sb.append("(project not found - leave app node pixel as-is)\n"); + return sb.toString(); + } + java.util.TreeSet reactors = project.getAvailableReactors(); + if (reactors == null || reactors.isEmpty()) { + sb.append("(none - leave app node pixel as a comment placeholder)\n"); + return sb.toString(); + } + for (String name : reactors) { + sb.append("- ").append(name).append("\n"); + } + } catch (Exception e) { + classLogger.warn("Failed to build reactor list for project {}", projectId, e); + sb.append("(reactor list unavailable - leave app node pixel as a comment placeholder)\n"); + } + return sb.toString(); + } + + /** + * Strips leading/trailing markdown code fences ({@code ```json ... ```} or {@code ``` ... ```}) + * that models sometimes add despite being instructed not to. + */ + public static String stripCodeFences(String raw) { + String trimmed = raw.strip(); + if (trimmed.startsWith("```")) { + int firstNewline = trimmed.indexOf('\n'); + if (firstNewline != -1) { + trimmed = trimmed.substring(firstNewline + 1); + } + if (trimmed.endsWith("```")) { + trimmed = trimmed.substring(0, trimmed.lastIndexOf("```")).strip(); + } + } + return trimmed; + } + + /** + * Validates that a generated document string is parseable JSON and contains the minimum + * required structure ({@code graph.nodes} must be non-empty). Throws on failure. + * + *

Uses Gson for parsing (consistent with the rest of the automation package) rather than + * {@code org.json}. Gson deserializes JSON objects as {@code Map} and arrays + * as {@code List}, so the casts below are safe for well-formed JSON. + */ + public static void validateGeneratedDoc(String raw) { + Map doc; + try { + doc = AutomationExecutionUtils.GSON.fromJson(raw, AutomationExecutionUtils.MAP_TYPE); + } catch (Exception e) { + classLogger.warn("Generated automation doc is not valid JSON (truncated): {}", + raw.length() > 500 ? raw.substring(0, 500) + "..." : raw, e); + throw new IllegalStateException( + "The AI model returned an invalid response. Please try again with a different description.", e); + } + if (doc == null || !(doc.get("graph") instanceof Map)) { + throw new IllegalStateException("Generated document is missing the 'graph' field."); + } + @SuppressWarnings("unchecked") + Map graph = (Map) doc.get("graph"); + if (!(graph.get("nodes") instanceof List)) { + throw new IllegalStateException("Generated graph is missing the 'nodes' array."); + } + @SuppressWarnings("unchecked") + List nodes = (List) graph.get("nodes"); + if (nodes.isEmpty()) { + throw new IllegalStateException("Generated graph has no nodes."); + } + } +} diff --git a/src/prerna/reactor/automation/utils/PixelExecutionUtils.java b/src/prerna/reactor/automation/utils/PixelExecutionUtils.java new file mode 100644 index 00000000000..2a231104379 --- /dev/null +++ b/src/prerna/reactor/automation/utils/PixelExecutionUtils.java @@ -0,0 +1,279 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation.utils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.om.Insight; +import prerna.om.ThreadStore; +import prerna.sablecc2.PixelRunner; +import prerna.sablecc2.om.PixelOperationType; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.sablecc2.om.task.ITask; +import prerna.reactor.automation.AutomationConstants; + +/** + * Shared utility for executing Pixel expressions and returning clean, serializable results. + * + *

Handles the common pitfalls of raw {@code insight.runPixel()} calls: + *

    + *
  • {@link ITask} materialization - query reactors return lazy cursors that must be collected
    • + *
  • Timeout enforcement - requests cancellation and prevents concurrent runs until the + * timed-out work has stopped
    • + *
  • Error extraction - detects {@link prerna.sablecc2.om.PixelOperationType#ERROR} in results
    • + *
  • Null-safe return - always returns a usable value
    • + *
+ */ +public final class PixelExecutionUtils { + + private static final Logger classLogger = LogManager.getLogger(PixelExecutionUtils.class); + + private PixelExecutionUtils() {} + + /** + * Executes a pixel expression and returns a clean, serializable result. + * + * @param insight the execution context (carries user, varstore, etc.) + * @param pixel the fully resolved pixel string to execute + * @param timeoutSeconds max execution time; 0 or negative means no timeout + * @return the pixel result as a serializable object (Map, List, String, Number, etc.), or null + * @throws AutomationNodeTimeoutException if execution exceeds the timeout + * @throws AutomationPixelException if the pixel returns an error result + */ + public static Object runAndCollect(Insight insight, String pixel, int timeoutSeconds) { + if (pixel == null || pixel.isBlank()) { + return null; + } + + classLogger.debug("Executing pixel (timeout={}s): {}", timeoutSeconds, + pixel.length() > 200 ? pixel.substring(0, 200) + "..." : pixel); + + NounMetadata result; + if (timeoutSeconds > 0) { + result = executeWithTimeout(insight, pixel, timeoutSeconds); + } else { + result = executeDirectly(insight, pixel); + } + + if (result == null) { + return null; + } + + checkForError(result, pixel); + return materializeValue(result); + } + + /** Overload with default timeout from {@link AutomationConstants#DEFAULT_TIMEOUT_SECONDS}. */ + public static Object runAndCollect(Insight insight, String pixel) { + return runAndCollect(insight, pixel, AutomationConstants.DEFAULT_TIMEOUT_SECONDS); + } + + // -- Private implementation ---------------------------------------------------- + + private static NounMetadata executeWithTimeout(Insight insight, String pixel, int timeoutSeconds) { + ExecutorService executor = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "automation-pixel-exec"); + t.setDaemon(true); + return t; + }); + + // ThreadStore is a plain ThreadLocal — not inherited by pool threads. + // Snapshot the caller's context so the worker thread has user/session/insight access. + Map callerContext = ThreadStore.getTheadMapObject(); + final Map contextSnapshot = + callerContext != null ? new HashMap<>(callerContext) : null; + AtomicReference activeRunner = new AtomicReference<>(); + AtomicBoolean timeoutRequested = new AtomicBoolean(false); + CountDownLatch executionTerminated = new CountDownLatch(1); + + try { + Callable task = () -> { + if (contextSnapshot != null && !contextSnapshot.isEmpty()) { + // ThreadStore.setThreadMapObject calls CURRENT.get() directly, which returns + // null on a fresh worker thread and would NPE. Call setInsightId("") first + // to force initialization of the ThreadLocal map; setThreadMapObject then + // overwrites all values including insightId with the captured context. + ThreadStore.setInsightId(""); + ThreadStore.setThreadMapObject(contextSnapshot); + } + try { + PixelRunner runner = insight.getPixelRunner(); + activeRunner.set(runner); + if (timeoutRequested.get()) { + runner.cancelRequest(); + } + return executeDirectly(insight, runner, pixel); + } finally { + ThreadStore.remove(); + executionTerminated.countDown(); + } + }; + + Future future = executor.submit(task); + try { + return future.get(timeoutSeconds, TimeUnit.SECONDS); + } catch (TimeoutException e) { + timeoutRequested.set(true); + PixelRunner runner = activeRunner.get(); + if (runner != null) { + runner.cancelRequest(); + } + future.cancel(true); + + // Future.cancel(true) only interrupts the worker. Some engine calls do not honor + // interruption immediately, so returning here would let AutomationRunEngine release + // its project lease while side effects may still be in flight. Hold the caller until + // the worker has actually terminated, then report the timeout. + awaitTermination(executionTerminated); + throw new AutomationNodeTimeoutException(pixel, timeoutSeconds); + } catch (ExecutionException e) { + Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) throw (RuntimeException) cause; + throw new IllegalStateException("Pixel execution failed: " + cause.getMessage(), cause); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Pixel execution interrupted", e); + } + } finally { + executor.shutdown(); + } + } + + private static NounMetadata executeDirectly(Insight insight, String pixel) { + return executeDirectly(insight, insight.getPixelRunner(), pixel); + } + + private static NounMetadata executeDirectly(Insight insight, PixelRunner runner, String pixel) { + List results = insight.runPixel(runner, pixel).getResults(); + if (results == null || results.isEmpty()) return null; + // For multi-statement pixels the meaningful result is the last one. + return results.get(results.size() - 1); + } + + private static void awaitTermination(CountDownLatch executionTerminated) { + boolean interrupted = false; + while (true) { + try { + executionTerminated.await(); + break; + } catch (InterruptedException e) { + interrupted = true; + } + } + if (interrupted) { + Thread.currentThread().interrupt(); + } + } + + private static void checkForError(NounMetadata result, String pixel) { + if (result.getOpType() != null && result.getOpType().contains(PixelOperationType.ERROR)) { + String errorMsg = result.getValue() != null ? result.getValue().toString() : "Unknown pixel error"; + throw new AutomationPixelException(pixel, errorMsg); + } + } + + private static Object materializeValue(NounMetadata result) { + Object value = result.getValue(); + if (value == null) return null; + + if (value instanceof ITask) { + try { + return ((ITask) value).collect(false); + } catch (Exception e) { + classLogger.error("Failed to materialize ITask: {}", e.getMessage(), e); + throw new IllegalStateException("Failed to materialize query result: " + e.getMessage(), e); + } + } + return value; + } + + // -- Exception types ----------------------------------------------------------- + + /** Thrown when a pixel execution exceeds its configured timeout. */ + public static class AutomationNodeTimeoutException extends RuntimeException { + private final int timeoutSeconds; + + public AutomationNodeTimeoutException(String pixel, int timeoutSeconds) { + super("Pixel execution timed out after " + timeoutSeconds + " seconds: " + + (pixel.length() > 100 ? pixel.substring(0, 100) + "..." : pixel)); + this.timeoutSeconds = timeoutSeconds; + } + + public int getTimeoutSeconds() { + return timeoutSeconds; + } + } + + /** Thrown when a pixel returns an ERROR operation type. */ + public static class AutomationPixelException extends RuntimeException { + private final String pixel; + + public AutomationPixelException(String pixel, String errorMessage) { + super(errorMessage); + this.pixel = pixel; + } + + public String getPixel() { + return pixel; + } + } + + /** + * Thrown mid-node when a cancellation request is detected during a blocking operation. + * + *

Using a distinct unchecked exception type (rather than a flag return value or a checked + * exception) lets nodes that loop internally - such as {@code WaitNodeExecutor} sleeping in + * chunks - abort cleanly without threading a cancellation result through every call frame. The + * caller ({@code AutomationRunEngine.executeSingleNode}) catches this type specifically and + * records the run as {@link AutomationConstants#STATUS_CANCELLED} instead of + * {@link AutomationConstants#STATUS_FAILED}, so the end-user sees the correct terminal state. + */ + public static class AutomationCancelledException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public AutomationCancelledException(String message) { + super(message); + } + } +} diff --git a/src/prerna/reactor/project/CreateProjectReactor.java b/src/prerna/reactor/project/CreateProjectReactor.java index c489ea1db79..eaa124bd402 100644 --- a/src/prerna/reactor/project/CreateProjectReactor.java +++ b/src/prerna/reactor/project/CreateProjectReactor.java @@ -131,6 +131,11 @@ public NounMetadata execute() { + "Use CreateNotebook(project='...') instead — it scaffolds the sample .ipynb " + "file that CreateProject skips."); } + if (projectType == IProject.PROJECT_TYPE.AUTOMATION) { + throw new IllegalArgumentException("CreateProject cannot create AUTOMATION-type projects. " + + "Use CreateAutomation(projectName='...') instead — it scaffolds the automation " + + "definition, configuration, and MCP tool metadata."); + } } String gitProvider = this.keyValue.get(this.keysToGet[index++]); String gitCloneUrl = this.keyValue.get(this.keysToGet[index++]); diff --git a/src/prerna/reactor/scheduler/SchedulerConstants.java b/src/prerna/reactor/scheduler/SchedulerConstants.java index aaa3e5e4b4d..f79530be26f 100644 --- a/src/prerna/reactor/scheduler/SchedulerConstants.java +++ b/src/prerna/reactor/scheduler/SchedulerConstants.java @@ -128,6 +128,7 @@ private SchedulerConstants() { public static final String VARCHAR_250 = "VARCHAR (250)"; public static final String VARCHAR_255 = "VARCHAR (255)"; public static final String VARCHAR_512 = "VARCHAR (512)"; + public static final String VARCHAR_2000 = "VARCHAR (2000)"; public static final String INTEGER = "INTEGER"; public static final String BOOLEAN = "BOOLEAN"; diff --git a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java index 9edd86100ea..bfbcf0a13db 100644 --- a/src/prerna/reactor/scheduler/SchedulerOwlCreator.java +++ b/src/prerna/reactor/scheduler/SchedulerOwlCreator.java @@ -124,7 +124,6 @@ import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_8; import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_80; import static prerna.reactor.scheduler.SchedulerConstants.VARCHAR_95; - import java.util.ArrayList; import java.util.Arrays; diff --git a/src/prerna/util/Constants.java b/src/prerna/util/Constants.java index 364da3b7925..03087e76bcd 100644 --- a/src/prerna/util/Constants.java +++ b/src/prerna/util/Constants.java @@ -1082,6 +1082,7 @@ public class Constants { public static final String MCP_DATABASE_MAKER = "database-maker"; public static final String MCP_REACTOR_HELP = "reactor-help"; public static final String MCP_BROWSER_AUTOMATION = "browser-automation"; + public static final String MCP_AUTOMATION = "automation-portal"; // system (platform) agent (workspace) names public static final String AGENT_APP_BUILDER = "app-builder"; diff --git a/src/prerna/util/SMSSWebWatcher.java b/src/prerna/util/SMSSWebWatcher.java index 41cb117976f..5defd38c90c 100644 --- a/src/prerna/util/SMSSWebWatcher.java +++ b/src/prerna/util/SMSSWebWatcher.java @@ -47,6 +47,7 @@ import prerna.masterdatabase.utility.MasterDatabaseUtility; import prerna.notifications.NotificationDbUtils; import prerna.prompt.PromptUtils; +import prerna.reactor.automation.AutomationDatabaseUtility; import prerna.reactor.scheduler.SchedulerDatabaseUtility; import prerna.theme.AbstractThemeUtils; import prerna.usertracking.UserTrackingUtils; @@ -212,6 +213,10 @@ public void init() { try { SystemEngineRegistry.loadSystemEngine(folderToWatch + "/" + fileNames[schedulerDbNameIndex]); SchedulerDatabaseUtility.startServer(); + // Automation tables live in the scheduler DB, so only initialize them + // after the scheduler DB has started successfully. + AutomationDatabaseUtility.initialize(); + AutomationDatabaseUtility.markStaleRunsInterrupted(); } catch (Exception e) { classLogger.error("Failed to load and start the scheduler database", e); } diff --git a/src/prerna/util/SystemDefaultEngines.java b/src/prerna/util/SystemDefaultEngines.java index 17060d6e8ab..11db564b4b7 100644 --- a/src/prerna/util/SystemDefaultEngines.java +++ b/src/prerna/util/SystemDefaultEngines.java @@ -60,7 +60,7 @@ public class SystemDefaultEngines { * must have a matching {@code project/platform__} folder. */ private static final List SYSTEM_MCPS = List.of(Constants.MCP_NODE_BUILDER, Constants.MCP_DATABASE_MAKER, - Constants.MCP_REACTOR_HELP, Constants.MCP_BROWSER_AUTOMATION); + Constants.MCP_REACTOR_HELP, Constants.MCP_BROWSER_AUTOMATION, Constants.MCP_AUTOMATION); /** * Subset of {@link #SYSTEM_MCPS} seeded onto system agent workspaces. This is diff --git a/src/prerna/util/SystemEngineRegistry.java b/src/prerna/util/SystemEngineRegistry.java index 46ca3f33c56..9ce0068afc3 100644 --- a/src/prerna/util/SystemEngineRegistry.java +++ b/src/prerna/util/SystemEngineRegistry.java @@ -86,8 +86,8 @@ public final class SystemEngineRegistry { private static final Set LOCAL_MASTER_DB_ALLOWED = Set.of("prerna.auth", "prerna.masterdatabase", "prerna.reactor.masterdatabase", "prerna.reactor.utils", "prerna.util", "prerna.web.conf"); - private static final Set SCHEDULER_DB_ALLOWED = Set.of("prerna.auth", "prerna.reactor.scheduler", - "prerna.util", "prerna.web.conf"); + private static final Set SCHEDULER_DB_ALLOWED = Set.of("prerna.auth", "prerna.reactor.automation", + "prerna.reactor.scheduler", "prerna.util", "prerna.web.conf"); private static final Set THEMING_DB_ALLOWED = Set.of("prerna.auth", "prerna.theme", "prerna.util", "prerna.web.conf"); diff --git a/test/prerna/reactor/automation/AutomationDefinitionValidatorTest.java b/test/prerna/reactor/automation/AutomationDefinitionValidatorTest.java new file mode 100644 index 00000000000..e0b680474cf --- /dev/null +++ b/test/prerna/reactor/automation/AutomationDefinitionValidatorTest.java @@ -0,0 +1,112 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.automation; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +class AutomationDefinitionValidatorTest { + + @Test + void validatesTriggerOnlyDefinitionWithoutRequiringRunnableSteps() { + AutomationDefinitionValidator.ValidatedDefinition definition = + AutomationDefinitionValidator.parseAndValidate(document( + "[{\"id\":\"trigger\",\"type\":\"trigger\"}]", "[]")); + + assertEquals(AutomationConstants.DOC_CURRENT_VERSION, definition.getVersion()); + assertEquals(1, definition.getNodes().size()); + } + + @Test + void canonicalizesEquivalentDocumentsToTheSameSnapshotAndHash() { + String first = """ + {"version":1,"graph":{"nodes":[{"id":"trigger","type":"trigger"},{"id":"wait","type":"wait"}], + "edges":[{"source":"trigger","target":"wait"}]}}"""; + String second = """ + {"graph":{"edges":[{"target":"wait","source":"trigger"}], + "nodes":[{"type":"trigger","id":"trigger"},{"type":"wait","id":"wait"}]},"version":1}"""; + + AutomationDefinitionValidator.ValidatedDefinition firstDefinition = + AutomationDefinitionValidator.parseAndValidate(first); + AutomationDefinitionValidator.ValidatedDefinition secondDefinition = + AutomationDefinitionValidator.parseAndValidate(second); + + assertEquals(firstDefinition.getSnapshot(), secondDefinition.getSnapshot()); + assertEquals(firstDefinition.getHash(), secondDefinition.getHash()); + } + + @Test + void rejectsInvalidVersionsAndNodes() { + assertThrows(IllegalArgumentException.class, () -> + AutomationDefinitionValidator.parseAndValidate( + "{\"version\":2,\"graph\":{\"nodes\":[],\"edges\":[]}}")); + assertThrows(IllegalArgumentException.class, () -> + AutomationDefinitionValidator.parseAndValidate(document( + "[{\"id\":\"trigger\",\"type\":\"trigger\"},{\"id\":\"trigger\",\"type\":\"wait\"}]", "[]"))); + assertThrows(IllegalArgumentException.class, () -> + AutomationDefinitionValidator.parseAndValidate(document( + "[{\"id\":\"trigger\",\"type\":\"trigger\"},{\"id\":\"unknown\",\"type\":\"unknown\"}]", "[]"))); + } + + @Test + void rejectsInvalidEdgesAndCycles() { + assertThrows(IllegalArgumentException.class, () -> + AutomationDefinitionValidator.parseAndValidate(document( + "[{\"id\":\"trigger\",\"type\":\"trigger\"}]", "[{\"source\":\"trigger\",\"target\":\"missing\"}]"))); + assertThrows(IllegalArgumentException.class, () -> + AutomationDefinitionValidator.parseAndValidate(document( + "[{\"id\":\"trigger\",\"type\":\"trigger\"}]", "[{\"source\":\"trigger\",\"target\":\"trigger\"}]"))); + assertThrows(IllegalArgumentException.class, () -> + AutomationDefinitionValidator.parseAndValidate(document( + "[{\"id\":\"trigger\",\"type\":\"trigger\"},{\"id\":\"one\",\"type\":\"wait\"},{\"id\":\"two\",\"type\":\"wait\"}]", + "[{\"source\":\"trigger\",\"target\":\"one\"},{\"source\":\"one\",\"target\":\"two\"},{\"source\":\"two\",\"target\":\"one\"}]"))); + } + + @Test + void ordersNodesByDependenciesWhilePreservingReadyNodeDocumentOrder() { + AutomationDefinitionValidator.ValidatedDefinition definition = + AutomationDefinitionValidator.parseAndValidate(document( + "[{\"id\":\"trigger\",\"type\":\"trigger\"},{\"id\":\"second\",\"type\":\"wait\"}," + + "{\"id\":\"first\",\"type\":\"wait\"},{\"id\":\"join\",\"type\":\"wait\"}]", + "[{\"source\":\"trigger\",\"target\":\"first\"},{\"source\":\"trigger\",\"target\":\"second\"}," + + "{\"source\":\"first\",\"target\":\"join\"},{\"source\":\"second\",\"target\":\"join\"}]")); + + List nodeIds = definition.getExecutionOrder().stream() + .map(node -> (String) node.get(AutomationConstants.NODE_FIELD_ID)) + .toList(); + + assertEquals(List.of("trigger", "second", "first", "join"), nodeIds); + } + + private static String document(String nodes, String edges) { + return "{\"version\":1,\"graph\":{\"nodes\":" + nodes + ",\"edges\":" + edges + "}}"; + } +}