From ab1498ddf66fe7759f750247da39df618ff06879 Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Tue, 28 Jul 2026 11:23:59 -0400 Subject: [PATCH 1/3] feat: flyway style migration --- .../engine/impl/rdbms/RDBMSNativeEngine.java | 20 ++ .../impl/rdbms/migration/MigrationFile.java | 67 ++++ .../rdbms/migration/MigrationFileUtils.java | 215 +++++++++++ .../migration/MigrationHistoryRecord.java | 102 ++++++ .../migration/MigrationHistoryUtils.java | 196 ++++++++++ .../impl/rdbms/migration/MigrationStatus.java | 110 ++++++ .../rdbms/migration/MigrationStatusUtils.java | 126 +++++++ .../migration/RdbmsMigrationOwlSyncUtils.java | 236 ++++++++++++ .../migration/SchemaMigrationException.java | 52 +++ .../rdbms/migration/SchemaMigrationLock.java | 340 ++++++++++++++++++ .../SchemaMigrationLockTimeoutException.java | 46 +++ .../migration/SchemaMigrationRunner.java | 277 ++++++++++++++ .../GetEngineMigrationsEnabledReactor.java | 89 +++++ .../ListEngineMigrationsReactor.java | 116 ++++++ .../migration/SaveEngineMigrationReactor.java | 195 ++++++++++ src/prerna/util/Constants.java | 3 + src/prerna/util/FileSystemUtil.java | 29 +- 17 files changed, 2207 insertions(+), 12 deletions(-) create mode 100644 src/prerna/engine/impl/rdbms/migration/MigrationFile.java create mode 100644 src/prerna/engine/impl/rdbms/migration/MigrationFileUtils.java create mode 100644 src/prerna/engine/impl/rdbms/migration/MigrationHistoryRecord.java create mode 100644 src/prerna/engine/impl/rdbms/migration/MigrationHistoryUtils.java create mode 100644 src/prerna/engine/impl/rdbms/migration/MigrationStatus.java create mode 100644 src/prerna/engine/impl/rdbms/migration/MigrationStatusUtils.java create mode 100644 src/prerna/engine/impl/rdbms/migration/RdbmsMigrationOwlSyncUtils.java create mode 100644 src/prerna/engine/impl/rdbms/migration/SchemaMigrationException.java create mode 100644 src/prerna/engine/impl/rdbms/migration/SchemaMigrationLock.java create mode 100644 src/prerna/engine/impl/rdbms/migration/SchemaMigrationLockTimeoutException.java create mode 100644 src/prerna/engine/impl/rdbms/migration/SchemaMigrationRunner.java create mode 100644 src/prerna/reactor/database/migration/GetEngineMigrationsEnabledReactor.java create mode 100644 src/prerna/reactor/database/migration/ListEngineMigrationsReactor.java create mode 100644 src/prerna/reactor/database/migration/SaveEngineMigrationReactor.java diff --git a/src/prerna/engine/impl/rdbms/RDBMSNativeEngine.java b/src/prerna/engine/impl/rdbms/RDBMSNativeEngine.java index cf4e9634b5b..82d4f14dece 100644 --- a/src/prerna/engine/impl/rdbms/RDBMSNativeEngine.java +++ b/src/prerna/engine/impl/rdbms/RDBMSNativeEngine.java @@ -27,6 +27,7 @@ *******************************************************************************/ package prerna.engine.impl.rdbms; +import java.io.File; import java.io.IOException; import java.sql.Clob; import java.sql.Connection; @@ -56,6 +57,8 @@ import prerna.engine.api.IRDBMSEngine; import prerna.engine.impl.AbstractDatabaseEngine; import prerna.engine.impl.SmssUtilities; +import prerna.engine.impl.rdbms.migration.MigrationFileUtils; +import prerna.engine.impl.rdbms.migration.SchemaMigrationRunner; import prerna.query.interpreters.IQueryInterpreter; import prerna.query.querystruct.SelectQueryStruct; import prerna.query.querystruct.filters.IQueryFilter; @@ -337,6 +340,23 @@ public void open(Properties smssProp) throws Exception { } catch (SQLException e) { classLogger.error("Failed to establish database connection during open: {}", e.getMessage(), e); } + + if (this.engineConnected && Boolean.parseBoolean(this.smssProp.getProperty(Constants.ENABLE_MIGRATIONS))) { + runPendingMigrations(); + } + } + + /** + * Runs any pending {@code V__.sql} files under this + * engine's own {@code assets/.migrations} folder. Unlike the connection + * failure above, a migration failure is intentionally allowed to propagate + * out of {@code open()} -- {@code Utility.loadEngine()} never registers an + * engine whose {@code open()} throws, so a bad migration leaves the engine + * fully unusable rather than usable with a half-migrated, unreliable schema. + */ + private void runPendingMigrations() { + File migrationsFolder = MigrationFileUtils.getMigrationsFolder(this); + SchemaMigrationRunner.runPendingMigrations(this, migrationsFolder); } /** diff --git a/src/prerna/engine/impl/rdbms/migration/MigrationFile.java b/src/prerna/engine/impl/rdbms/migration/MigrationFile.java new file mode 100644 index 00000000000..5d1c1334b95 --- /dev/null +++ b/src/prerna/engine/impl/rdbms/migration/MigrationFile.java @@ -0,0 +1,67 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.engine.impl.rdbms.migration; + +/** + * Plain data holder for a single {@code V__.sql} file + * discovered under an engine's {@code assets/.migrations} folder. This is only + * ever produced by {@link MigrationFileUtils#scanPendingMigrations} — files + * are UI-created only, never hand-edited, so this is intentionally not a + * mutable builder. + */ +public class MigrationFile { + + private final String version; + private final String description; + private final String fileName; + private final String sqlContent; + + public MigrationFile(String version, String description, String fileName, String sqlContent) { + this.version = version; + this.description = description; + this.fileName = fileName; + this.sqlContent = sqlContent; + } + + public String getVersion() { + return version; + } + + public String getDescription() { + return description; + } + + public String getFileName() { + return fileName; + } + + public String getSqlContent() { + return sqlContent; + } + +} diff --git a/src/prerna/engine/impl/rdbms/migration/MigrationFileUtils.java b/src/prerna/engine/impl/rdbms/migration/MigrationFileUtils.java new file mode 100644 index 00000000000..19edad03558 --- /dev/null +++ b/src/prerna/engine/impl/rdbms/migration/MigrationFileUtils.java @@ -0,0 +1,215 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.engine.impl.rdbms.migration; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.engine.api.IEngine; +import prerna.engine.api.IRDBMSEngine; +import prerna.util.EngineUtility; + +/** + * Scans an engine's {@code assets/.migrations} folder for versioned SQL + * migration files and computes their checksums. Files are only ever created + * by the Migrations UI tab (never hand-edited), but this still validates the + * naming convention defensively — e.g. a folder restored from an old export + * — and simply ignores/logs anything that doesn't match, the same way other + * asset folders in SEMOSS are scanned (see {@code FaissDatabaseEngine}'s + * {@code .pkl} filter, {@code ListPlaywrightScriptsReactor}'s {@code .json} + * filter). + */ +public final class MigrationFileUtils { + + private static final Logger classLogger = LogManager.getLogger(MigrationFileUtils.class); + + /** {@code V__.sql} — matches Flyway's versioned migration convention. */ + private static final Pattern MIGRATION_FILE_PATTERN = Pattern + .compile("^V(\\d+(?:\\.\\d+)*)__(.+)\\.sql$", Pattern.CASE_INSENSITIVE); + + private MigrationFileUtils() { + // utility class + } + + /** + * @param engine the engine whose migrations folder should be resolved + * @return the engine's own {@code assets/.migrations} folder -- shared + * resolution logic used both by {@code RDBMSNativeEngine.open()} + * and by the read-only status reactor, so the two never drift + * apart on where this folder lives + */ + public static File getMigrationsFolder(IRDBMSEngine engine) { + String assetsFolder = EngineUtility.getSpecificEngineAssetsFolder(IEngine.CATALOG_TYPE.DATABASE, + engine.getEngineId(), engine.getEngineName()); + return new File(assetsFolder, ".migrations"); + } + + /** + * @param migrationsFolder the engine's {@code assets/.migrations} folder + * @return every validly-named migration file in the folder, ordered by + * version ascending (dotted versions compared numerically, segment + * by segment — e.g. {@code V2} before {@code V2.1} before + * {@code V10}) + */ + public static List scanMigrationsFolder(File migrationsFolder) { + List migrations = new ArrayList<>(); + if (migrationsFolder == null || !migrationsFolder.exists() || !migrationsFolder.isDirectory()) { + return migrations; + } + + File[] files = migrationsFolder.listFiles(); + if (files == null) { + return migrations; + } + + for (File file : files) { + if (!file.isFile()) { + continue; + } + MigrationFile migration = parseMigrationFile(file); + if (migration != null) { + migrations.add(migration); + } + } + + migrations.sort(Comparator.comparing(MigrationFile::getVersion, MigrationFileUtils::compareVersions)); + return migrations; + } + + private static MigrationFile parseMigrationFile(File file) { + Matcher matcher = MIGRATION_FILE_PATTERN.matcher(file.getName()); + if (!matcher.matches()) { + classLogger.warn( + "Ignoring file '{}' in migrations folder -- does not match the required " + + "V__.sql naming convention", + file.getName()); + return null; + } + + String version = matcher.group(1); + String description = matcher.group(2); + String sqlContent; + try { + sqlContent = Files.readString(file.toPath(), StandardCharsets.UTF_8); + } catch (IOException e) { + classLogger.error("Failed to read migration file '{}'.", file.getName(), e); + throw new SchemaMigrationException("Unable to read migration file " + file.getName(), e); + } + return new MigrationFile(version, description, file.getName(), sqlContent); + } + + /** + * Compares two dotted version strings numerically, segment by segment (e.g. + * {@code "2"} < {@code "2.1"} < {@code "10"}), so folder/lexicographic + * ordering never produces the wrong execution order. + */ + public static int compareVersions(String left, String right) { + String[] leftParts = left.split("\\."); + String[] rightParts = right.split("\\."); + int maxLength = Math.max(leftParts.length, rightParts.length); + for (int i = 0; i < maxLength; i++) { + int leftSegment = i < leftParts.length ? Integer.parseInt(leftParts[i]) : 0; + int rightSegment = i < rightParts.length ? Integer.parseInt(rightParts[i]) : 0; + int comparison = Integer.compare(leftSegment, rightSegment); + if (comparison != 0) { + return comparison; + } + } + return 0; + } + + /** + * @param sqlContent the migration file's content + * @return a hex-encoded SHA-256 checksum, used to detect a version's file + * being edited after it already ran + */ + public static String computeChecksum(String sqlContent) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] hash = digest.digest(sqlContent.getBytes(StandardCharsets.UTF_8)); + StringBuilder hex = new StringBuilder(hash.length * 2); + for (byte b : hash) { + hex.append(String.format("%02x", b)); + } + return hex.toString(); + } catch (NoSuchAlgorithmException e) { + // SHA-256 is a guaranteed JDK algorithm; this cannot actually happen + throw new IllegalStateException(e); + } + } + + /** + * Minimal statement splitter: one statement per semicolon-terminated group + * of lines, ignoring blank lines and {@code --} comment lines. Does not + * handle semicolons inside string literals or stored-procedure bodies -- + * out of scope for v1 (raw DDL/DML content only), carried over as-is from + * the earlier migration-poc design. + */ + public static List splitStatements(String sql) { + List statements = new ArrayList<>(); + StringBuilder current = new StringBuilder(); + for (String line : sql.split("\n")) { + String trimmedLine = line.trim(); + if (trimmedLine.isEmpty() || trimmedLine.startsWith("--")) { + continue; + } + current.append(line).append('\n'); + if (trimmedLine.endsWith(";")) { + addStatement(statements, current.toString()); + current.setLength(0); + } + } + if (current.length() > 0) { + addStatement(statements, current.toString()); + } + return statements; + } + + private static void addStatement(List statements, String rawStatement) { + String statement = rawStatement.trim(); + if (statement.endsWith(";")) { + statement = statement.substring(0, statement.length() - 1).trim(); + } + if (!statement.isEmpty()) { + statements.add(statement); + } + } + +} diff --git a/src/prerna/engine/impl/rdbms/migration/MigrationHistoryRecord.java b/src/prerna/engine/impl/rdbms/migration/MigrationHistoryRecord.java new file mode 100644 index 00000000000..cadbed896f2 --- /dev/null +++ b/src/prerna/engine/impl/rdbms/migration/MigrationHistoryRecord.java @@ -0,0 +1,102 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.engine.impl.rdbms.migration; + +import java.sql.Timestamp; + +/** + * Plain data holder for a single row of {@code SEMOSS_SCHEMA_HISTORY} — the + * run outcome for one specific migration version, recorded inside the target + * engine's own database (not centrally, unlike the earlier migration-poc + * design). One row per attempt; a re-run after a fix is a new row, not an + * update of the old one. + */ +public class MigrationHistoryRecord { + + /** Attribution used for every automatic run — there is no user/{@code Insight} at engine-open() time. */ + public static final String SYSTEM_APPLIED_BY = "SYSTEM"; + + private final String version; + private final String scriptName; + private final String checksum; + private final String appliedBy; + private final Timestamp appliedOn; + private final long executionTimeMs; + private final boolean success; + private final String description; + + public MigrationHistoryRecord(String version, String scriptName, String checksum, String appliedBy, + Timestamp appliedOn, long executionTimeMs, boolean success, String description) { + this.version = version; + this.scriptName = scriptName; + this.checksum = checksum; + this.appliedBy = appliedBy; + this.appliedOn = appliedOn; + this.executionTimeMs = executionTimeMs; + this.success = success; + this.description = description; + } + + public String getVersion() { + return version; + } + + public String getScriptName() { + return scriptName; + } + + public String getChecksum() { + return checksum; + } + + public String getAppliedBy() { + return appliedBy; + } + + public Timestamp getAppliedOn() { + return appliedOn; + } + + public long getExecutionTimeMs() { + return executionTimeMs; + } + + public boolean isSuccess() { + return success; + } + + /** + * On a failed run, this carries the failure reason (repurposed, same + * convention as the earlier migration-poc design); on a successful run it + * is {@code null}. + */ + public String getDescription() { + return description; + } + +} diff --git a/src/prerna/engine/impl/rdbms/migration/MigrationHistoryUtils.java b/src/prerna/engine/impl/rdbms/migration/MigrationHistoryUtils.java new file mode 100644 index 00000000000..869e388b130 --- /dev/null +++ b/src/prerna/engine/impl/rdbms/migration/MigrationHistoryUtils.java @@ -0,0 +1,196 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.engine.impl.rdbms.migration; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.engine.api.IRDBMSEngine; +import prerna.util.ConnectionUtils; +import prerna.util.sql.AbstractSqlQueryUtil; + +/** + * Reads and writes {@code SEMOSS_SCHEMA_HISTORY} -- the run-history table + * that lives inside the target engine's own database (not centrally, unlike + * the earlier migration-poc design), so it travels naturally with a DB + * export/import. Table creation follows the same + * check-then-create-if-not-exists convention every other SEMOSS internal + * table uses (see {@code AbstractSecurityUtils#loadSecurityDatabase}). + */ +public final class MigrationHistoryUtils { + + private static final Logger classLogger = LogManager.getLogger(MigrationHistoryUtils.class); + + /** Public so other migration-package classes (e.g. the OWL sync utility) can exclude this reserved table by name. */ + public static final String HISTORY_TABLE = "SEMOSS_SCHEMA_HISTORY"; + + private static final String[] COL_NAMES = { "VERSION", "SCRIPTNAME", "CHECKSUM", "APPLIEDBY", "APPLIEDON", + "EXECUTIONTIMEMS", "SUCCESS", "DESCRIPTION" }; + + private MigrationHistoryUtils() { + // utility class + } + + /** + * Idempotent check-then-create for {@code SEMOSS_SCHEMA_HISTORY} inside the + * given engine's own database. + * + * @param engine the engine whose database should have the history table + */ + public static void ensureHistoryTable(IRDBMSEngine engine) { + AbstractSqlQueryUtil queryUtil = engine.getQueryUtil(); + String[] types = { "VARCHAR(255)", "VARCHAR(255)", "VARCHAR(255)", "VARCHAR(255)", + queryUtil.getDateWithTimeDataType(), "BIGINT", queryUtil.getBooleanDataTypeName(), "VARCHAR(2000)" }; + + Connection conn = null; + try { + conn = engine.getConnection(); + if (!queryUtil.tableExists(conn, HISTORY_TABLE, engine.getDatabase(), engine.getSchema())) { + String createSql = queryUtil.createTable(HISTORY_TABLE, COL_NAMES, types); + classLogger.info("Creating migration history table for engine '{}' with sql {}", engine.getEngineId(), + createSql); + engine.insertData(createSql); + } + } catch (Exception e) { + classLogger.error("Failed to ensure migration history table exists for engine '{}'.", engine.getEngineId(), + e); + throw new SchemaMigrationException( + "Unable to create or verify " + HISTORY_TABLE + " for engine " + engine.getEngineId(), e); + } finally { + ConnectionUtils.closeAllConnectionsIfPooling(engine, conn); + } + } + + /** + * @param engine the engine to read migration history from + * @return every recorded run attempt, ordered by version ascending + */ + public static List getHistory(IRDBMSEngine engine) { + // Deliberately raw JDBC, not SelectQueryStruct: SEMOSS_SCHEMA_HISTORY *is* + // registered as an OWL concept (see RdbmsMigrationOwlSyncUtils), but only + // after the first migration actually runs and its post-run OWL sync + // fires. This method can be called before that ever happens (e.g. right + // after ENABLE_MIGRATIONS is turned on, before any migration has run) -- + // SelectQueryStruct resolves column selectors against OWL-registered + // physical URIs, so relying on it here would fail during that bootstrap + // window. Raw JDBC keeps this read correct regardless of OWL sync + // timing. + String selectSql = "SELECT VERSION, SCRIPTNAME, CHECKSUM, APPLIEDBY, APPLIEDON, EXECUTIONTIMEMS, SUCCESS, " + + "DESCRIPTION FROM " + HISTORY_TABLE + " ORDER BY VERSION ASC"; + Connection conn = null; + List records = new ArrayList<>(); + try { + conn = engine.getConnection(); + if (!engine.getQueryUtil().tableExists(conn, HISTORY_TABLE, engine.getDatabase(), engine.getSchema())) { + // nothing has ever run against this engine yet + return records; + } + try (PreparedStatement ps = conn.prepareStatement(selectSql); ResultSet rs = ps.executeQuery()) { + while (rs.next()) { + records.add(new MigrationHistoryRecord(rs.getString("VERSION"), rs.getString("SCRIPTNAME"), + rs.getString("CHECKSUM"), rs.getString("APPLIEDBY"), rs.getTimestamp("APPLIEDON"), + rs.getLong("EXECUTIONTIMEMS"), rs.getBoolean("SUCCESS"), rs.getString("DESCRIPTION"))); + } + } + } catch (SQLException e) { + classLogger.error("Failed to read migration history for engine '{}'.", engine.getEngineId(), e); + throw new SchemaMigrationException("Unable to read migration history for engine " + engine.getEngineId(), + e); + } finally { + ConnectionUtils.closeAllConnectionsIfPooling(engine, conn); + } + return records; + } + + /** + * Records a run outcome using a connection/transaction the caller already + * owns and will commit itself -- used on the success path so the + * migration's SQL and its history row commit atomically together. + * Confirmed from Flyway's own source + * ({@code SchemaHistory.java}: "a migration failure automatically triggers + * a rollback of all changes, including the ones in the schema history + * table") that this same-transaction pairing is the real behavior worth + * matching, not two independent commits. + * + * @param conn an open connection/transaction the caller controls; this + * method does not commit or close it + * @param record the run outcome to record + */ + public static void insertHistoryRow(Connection conn, MigrationHistoryRecord record) throws SQLException { + String insertSql = "INSERT INTO " + HISTORY_TABLE + + " (VERSION, SCRIPTNAME, CHECKSUM, APPLIEDBY, APPLIEDON, EXECUTIONTIMEMS, SUCCESS, DESCRIPTION) " + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)"; + try (PreparedStatement ps = conn.prepareStatement(insertSql)) { + int index = 1; + ps.setString(index++, record.getVersion()); + ps.setString(index++, record.getScriptName()); + ps.setString(index++, record.getChecksum()); + ps.setString(index++, record.getAppliedBy()); + ps.setTimestamp(index++, record.getAppliedOn()); + ps.setLong(index++, record.getExecutionTimeMs()); + ps.setBoolean(index++, record.isSuccess()); + ps.setString(index++, record.getDescription()); + ps.execute(); + } + } + + /** + * Records a run outcome on its own, independently-committed + * connection/transaction -- used on the failure path, where the + * migration's own transaction has already been rolled back and recording + * the failure needs a fresh connection to actually persist. + * + * @param engine the engine whose history table this run outcome belongs to + * @param record the run outcome to record + */ + public static void recordMigration(IRDBMSEngine engine, MigrationHistoryRecord record) { + Connection conn = null; + try { + conn = engine.getConnection(); + insertHistoryRow(conn, record); + if (!conn.getAutoCommit()) { + conn.commit(); + } + } catch (Exception e) { + classLogger.error("Failed to record migration history for engine '{}', version '{}'.", + engine.getEngineId(), record.getVersion(), e); + throw new SchemaMigrationException( + "Unable to record migration history for version " + record.getVersion(), e); + } finally { + ConnectionUtils.closeAllConnectionsIfPooling(engine, conn); + } + } + +} diff --git a/src/prerna/engine/impl/rdbms/migration/MigrationStatus.java b/src/prerna/engine/impl/rdbms/migration/MigrationStatus.java new file mode 100644 index 00000000000..7200e7e437c --- /dev/null +++ b/src/prerna/engine/impl/rdbms/migration/MigrationStatus.java @@ -0,0 +1,110 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.engine.impl.rdbms.migration; + +import java.sql.Timestamp; + +/** + * The derived, display-ready status of one migration version -- merges a + * {@link MigrationFile} discovered on disk with its most recent + * {@link MigrationHistoryRecord} row, the same way Flyway's {@code info} + * command reports migration state. Produced only by + * {@link MigrationStatusUtils#getStatus}. + */ +public class MigrationStatus { + + /** Mirrors the subset of Flyway's {@code info} states relevant to this design (no undo/repeatable support). */ + public enum State { + /** File exists, never run. */ + PENDING, + /** File exists, last run succeeded, checksum unchanged. */ + SUCCESS, + /** File exists, last run failed. */ + FAILED, + /** History row exists but the file is no longer present on disk. */ + MISSING, + /** File exists and previously succeeded, but its content has changed since. */ + OUTDATED + } + + private final String version; + private final String description; + private final String fileName; + private final State state; + private final String appliedBy; + private final Timestamp appliedOn; + private final long executionTimeMs; + private final String errorMessage; + + public MigrationStatus(String version, String description, String fileName, State state, String appliedBy, + Timestamp appliedOn, long executionTimeMs, String errorMessage) { + this.version = version; + this.description = description; + this.fileName = fileName; + this.state = state; + this.appliedBy = appliedBy; + this.appliedOn = appliedOn; + this.executionTimeMs = executionTimeMs; + this.errorMessage = errorMessage; + } + + public String getVersion() { + return version; + } + + public String getDescription() { + return description; + } + + /** {@code null} for a {@link State#MISSING} row -- there is no file to name. */ + public String getFileName() { + return fileName; + } + + public State getState() { + return state; + } + + public String getAppliedBy() { + return appliedBy; + } + + public Timestamp getAppliedOn() { + return appliedOn; + } + + public long getExecutionTimeMs() { + return executionTimeMs; + } + + /** Failure reason for {@link State#FAILED}, or an explanatory note for {@link State#OUTDATED}; otherwise {@code null}. */ + public String getErrorMessage() { + return errorMessage; + } + +} diff --git a/src/prerna/engine/impl/rdbms/migration/MigrationStatusUtils.java b/src/prerna/engine/impl/rdbms/migration/MigrationStatusUtils.java new file mode 100644 index 00000000000..b52f8ebd1b9 --- /dev/null +++ b/src/prerna/engine/impl/rdbms/migration/MigrationStatusUtils.java @@ -0,0 +1,126 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.engine.impl.rdbms.migration; + +import java.io.File; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import prerna.engine.api.IRDBMSEngine; + +/** + * Merges what's on disk ({@link MigrationFileUtils#scanMigrationsFolder}) with + * what's recorded in {@code SEMOSS_SCHEMA_HISTORY} + * ({@link MigrationHistoryUtils#getHistory}) into a single, display-ready + * status list -- the same merge Flyway's {@code info} command performs. + * Read-only; this does not run or modify anything, unlike + * {@link SchemaMigrationRunner}. + */ +public final class MigrationStatusUtils { + + private MigrationStatusUtils() { + // utility class + } + + /** + * @param engine the engine to report migration status for + * @param migrationsFolder the engine's {@code assets/.migrations} folder + * @return one {@link MigrationStatus} per version known either on disk or in + * history, ordered by version ascending + */ + public static List getStatus(IRDBMSEngine engine, File migrationsFolder) { + List files = MigrationFileUtils.scanMigrationsFolder(migrationsFolder); + Map latestHistoryByVersion = latestRecordPerVersion( + MigrationHistoryUtils.getHistory(engine)); + + List statuses = new ArrayList<>(); + Set fileVersions = new HashSet<>(); + for (MigrationFile file : files) { + fileVersions.add(file.getVersion()); + statuses.add(buildStatusForFile(file, latestHistoryByVersion.get(file.getVersion()))); + } + for (MigrationHistoryRecord record : latestHistoryByVersion.values()) { + if (!fileVersions.contains(record.getVersion())) { + statuses.add(buildStatusForMissingFile(record)); + } + } + + statuses.sort(Comparator.comparing(MigrationStatus::getVersion, MigrationFileUtils::compareVersions)); + return statuses; + } + + /** + * {@link SchemaMigrationRunner} records one row per attempt, never + * overwriting a prior one -- so a version can have several rows if it was + * retried after a failure. Only the most recent attempt matters for + * display. + */ + private static Map latestRecordPerVersion(List history) { + Map latest = new HashMap<>(); + for (MigrationHistoryRecord record : history) { + MigrationHistoryRecord current = latest.get(record.getVersion()); + if (current == null || record.getAppliedOn().after(current.getAppliedOn())) { + latest.put(record.getVersion(), record); + } + } + return latest; + } + + private static MigrationStatus buildStatusForFile(MigrationFile file, MigrationHistoryRecord record) { + if (record == null) { + return new MigrationStatus(file.getVersion(), file.getDescription(), file.getFileName(), + MigrationStatus.State.PENDING, null, null, 0L, null); + } + if (!record.isSuccess()) { + return new MigrationStatus(file.getVersion(), file.getDescription(), file.getFileName(), + MigrationStatus.State.FAILED, record.getAppliedBy(), record.getAppliedOn(), + record.getExecutionTimeMs(), record.getDescription()); + } + + String currentChecksum = MigrationFileUtils.computeChecksum(file.getSqlContent()); + boolean outdated = !currentChecksum.equals(record.getChecksum()); + MigrationStatus.State state = outdated ? MigrationStatus.State.OUTDATED : MigrationStatus.State.SUCCESS; + String note = outdated + ? "File content has changed since this version was applied -- checksum no longer matches" + : null; + return new MigrationStatus(file.getVersion(), file.getDescription(), file.getFileName(), state, + record.getAppliedBy(), record.getAppliedOn(), record.getExecutionTimeMs(), note); + } + + private static MigrationStatus buildStatusForMissingFile(MigrationHistoryRecord record) { + return new MigrationStatus(record.getVersion(), record.getScriptName(), null, MigrationStatus.State.MISSING, + record.getAppliedBy(), record.getAppliedOn(), record.getExecutionTimeMs(), + "File no longer exists in the migrations folder"); + } + +} diff --git a/src/prerna/engine/impl/rdbms/migration/RdbmsMigrationOwlSyncUtils.java b/src/prerna/engine/impl/rdbms/migration/RdbmsMigrationOwlSyncUtils.java new file mode 100644 index 00000000000..f4c6d20349b --- /dev/null +++ b/src/prerna/engine/impl/rdbms/migration/RdbmsMigrationOwlSyncUtils.java @@ -0,0 +1,236 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.engine.impl.rdbms.migration; + +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.algorithm.api.SemossDataType; +import prerna.engine.api.IRDBMSEngine; +import prerna.engine.impl.owl.WriteOWLEngine; +import prerna.reactor.database.upload.rdbms.RDBMSEngineCreationHelper; +import prerna.util.EngineSyncUtility; +import prerna.util.UploadUtilities; +import prerna.util.Utility; + +/** + * Reconciles SEMOSS's OWL metamodel with an engine's real JDBC schema after a + * migration runs raw SQL directly against the engine's connection. + *

+ * This is the same diff-based sync the Metadata tab's "Sync" button performs + * ({@code ExternalUpdateJdbcSchema} discovery -> {@code RdbmsExternalUpload} + * with {@code existing=true} -> automatic Local Master re-registration), but + * called as plain Java rather than through that security-checked reactor + * chain: there is no {@code Insight}/user at engine-{@code open()} time for + * those reactors' {@code userCanEditEngine} checks to run against. The + * discovery and OWL-write logic is reused directly: + *

    + *
  • {@link RDBMSEngineCreationHelper#getExistingRDBMSStructure(prerna.engine.api.IDatabaseEngine)} + * for JDBC introspection (identical dialect handling the reactor chain + * uses) — called with no table filter to discover the engine's full current + * schema.
    • + *
  • {@link UploadUtilities#getExistingMetamodel(prerna.engine.impl.owl.AbstractOWLEngine)} + * plus direct {@link WriteOWLEngine} calls for the diff-write, mirroring + * {@code RdbmsExternalUploadReactor#updateExistingDatabase}.
    • + *
+ * The Local Master DB re-registration step is not performed here — + * {@code Utility.loadEngine()} already calls + * {@code Utility.synchronizeEngineMetadata(engineId)} immediately after + * {@code open()} returns, for every {@code DATABASE}-type engine, by + * comparing the OWL file's on-disk timestamp against what Local Master has + * recorded. Since {@link WriteOWLEngine#export()} below touches that file, + * that pre-existing hook picks up the change automatically — no need to + * duplicate it. + *

+ * Table/column relationship (foreign key) diffing is add-only in this first + * pass -- newly discovered foreign keys are added, but ones removed by a + * migration are not yet pruned from the OWL. Flagged as a follow-up; it does + * not affect whether new/changed tables and columns are visible to + * {@code SelectQueryStruct}-based queries, which is the primary correctness + * requirement here. + */ +public final class RdbmsMigrationOwlSyncUtils { + + private static final Logger classLogger = LogManager.getLogger(RdbmsMigrationOwlSyncUtils.class); + + /** + * Only {@code SEMOSS_SCHEMA_LOCK} is excluded from OWL -- it's transient + * concurrency-control plumbing with no query/diagnostic value (unlike + * {@code SEMOSS_SCHEMA_HISTORY}, nobody wants to inspect lock rows). + * {@code SEMOSS_SCHEMA_HISTORY} is deliberately registered like any other + * table -- matching the precedent {@code PGVectorQueryUtil.createOWL()} + * already sets for engine-managed (not user-authored) tables living + * alongside the user's own data: PGVector's embeddings/metadata tables are + * added to OWL the same way, not hidden from the ER diagram. + */ + private static final Set RESERVED_TABLE_NAMES = Set.of(SchemaMigrationLock.LOCK_TABLE); + + private RdbmsMigrationOwlSyncUtils() { + // utility class + } + + /** + * @param engine the engine whose OWL should be reconciled with its current + * JDBC schema + * @throws SchemaMigrationException if the sync fails -- by design this + * propagates out of + * {@code open()} rather than leaving the + * engine open with a stale OWL that + * {@code SelectQueryStruct} would resolve + * incorrectly against + */ + public static void syncOwlAfterMigration(IRDBMSEngine engine) { + try (WriteOWLEngine owlEngine = engine.getOWLEngineFactory().getWriteOWL()) { + Map> existingMetamodel = UploadUtilities + .getExistingMetamodel(owlEngine); + Map> newStructure = RDBMSEngineCreationHelper + .getExistingRDBMSStructure(engine); + RESERVED_TABLE_NAMES.forEach(newStructure::remove); + + if (existingMetamodel.equals(newStructure)) { + return; + } + + removeStaleConceptsAndProps(owlEngine, existingMetamodel, newStructure); + addNewConceptsAndProps(owlEngine, existingMetamodel, newStructure); + addNewRelationships(engine, owlEngine, newStructure); + + + owlEngine.commit(); + owlEngine.export(); + EngineSyncUtility.clearEngineCache(engine.getEngineId()); + } catch (Exception e) { + classLogger.error("Failed to sync OWL metadata for engine '{}' after migration run.", engine.getEngineId(), + e); + throw new SchemaMigrationException("Unable to sync OWL metadata for engine " + engine.getEngineId(), e); + } + } + + private static void removeStaleConceptsAndProps(WriteOWLEngine owlEngine, + Map> existingMetamodel, Map> newStructure) { + existingMetamodel.forEach((existingTableName, existingColumns) -> { + if (!newStructure.containsKey(existingTableName)) { + classLogger.info("Removing table '{}' from owl -- no longer present in the JDBC schema", + Utility.cleanLogString(existingTableName)); + owlEngine.removeConcept(existingTableName); + return; + } + Map newColumns = newStructure.get(existingTableName); + existingColumns.forEach((existingColumnName, existingDataType) -> { + String newDataType = newColumns.get(existingColumnName); + if (newDataType == null + || SemossDataType.convertStringToDataType(newDataType) != existingDataType) { + classLogger.info("Removing column '{}' for table '{}' from owl", + Utility.cleanLogString(existingColumnName), Utility.cleanLogString(existingTableName)); + owlEngine.removeProp(existingTableName, existingColumnName); + } + }); + }); + } + + private static void addNewConceptsAndProps(WriteOWLEngine owlEngine, + Map> existingMetamodel, Map> newStructure) { + newStructure.forEach((newTableName, newColumns) -> { + boolean isNewTable = !existingMetamodel.containsKey(newTableName); + if (isNewTable) { + classLogger.info("Adding table '{}' to owl", Utility.cleanLogString(newTableName)); + owlEngine.addConcept(newTableName, null, null); + } + Map existingColumns = existingMetamodel.get(newTableName); + newColumns.forEach((newColumnName, newDataType) -> { + boolean columnAlreadyPresent = existingColumns != null && existingColumns.containsKey(newColumnName) + && SemossDataType.convertStringToDataType(newDataType) == existingColumns.get(newColumnName); + if (!columnAlreadyPresent) { + classLogger.info("Adding column '{}' to table '{}' in owl", + Utility.cleanLogString(newColumnName), Utility.cleanLogString(newTableName)); + owlEngine.addProp(newTableName, newColumnName, newDataType, null, null); + } + }); + }); + } + + /** + * Add-only: newly discovered foreign keys are written as OWL relations. + * Removed foreign keys are not pruned in this first pass (see class + * Javadoc). + */ + private static void addNewRelationships(IRDBMSEngine engine, WriteOWLEngine owlEngine, + Map> newStructure) { + Connection conn = null; + try { + conn = engine.getConnection(); + DatabaseMetaData meta = conn.getMetaData(); + String catalog = engine.getQueryUtil().getDatabaseMetadataCatalogFilter(); + if (catalog == null) { + catalog = conn.getCatalog(); + } + String schema = engine.getQueryUtil().getDatabaseMetadataSchemaFilter(); + if (schema == null) { + schema = engine.getSchema(); + } + + for (String tableName : newStructure.keySet()) { + addRelationshipsForTable(owlEngine, meta, catalog, schema, tableName); + } + } catch (SQLException e) { + classLogger.error("Failed to discover foreign key relationships for engine '{}'.", engine.getEngineId(), + e); + throw new SchemaMigrationException( + "Unable to discover foreign key relationships for engine " + engine.getEngineId(), e); + } finally { + if (conn != null && engine.isConnectionPooling()) { + try { + conn.close(); + } catch (SQLException e) { + classLogger.error("Failed to close connection after discovering relationships.", e); + } + } + } + } + + private static void addRelationshipsForTable(WriteOWLEngine owlEngine, DatabaseMetaData meta, String catalog, + String schema, String tableName) throws SQLException { + try (ResultSet exportedKeys = meta.getExportedKeys(catalog, schema, tableName)) { + while (exportedKeys.next()) { + String toTable = exportedKeys.getString("FKTABLE_NAME"); + String fromCol = exportedKeys.getString("PKCOLUMN_NAME"); + String toCol = exportedKeys.getString("FKCOLUMN_NAME"); + owlEngine.addRelation(tableName, toTable, fromCol + "." + toCol); + } + } + } + +} diff --git a/src/prerna/engine/impl/rdbms/migration/SchemaMigrationException.java b/src/prerna/engine/impl/rdbms/migration/SchemaMigrationException.java new file mode 100644 index 00000000000..d4866a5ea35 --- /dev/null +++ b/src/prerna/engine/impl/rdbms/migration/SchemaMigrationException.java @@ -0,0 +1,52 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.engine.impl.rdbms.migration; + +/** + * Thrown when a pending migration cannot be safely run -- a failed SQL + * migration, an out-of-order version, a checksum mismatch against an + * already-applied version, or a folder/history read failure. Always thrown + * from inside {@code IRDBMSEngine.open(Properties)}, so by design it + * propagates out of {@code open()} and the engine is never registered + * (confirmed in {@code Utility.loadEngine} -- a thrown {@code open()} + * exception nulls out the engine before it reaches {@code DIHelper}), rather + * than leaving a partially-migrated engine usable. + */ +public class SchemaMigrationException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public SchemaMigrationException(String message) { + super(message); + } + + public SchemaMigrationException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/src/prerna/engine/impl/rdbms/migration/SchemaMigrationLock.java b/src/prerna/engine/impl/rdbms/migration/SchemaMigrationLock.java new file mode 100644 index 00000000000..860d9948f0d --- /dev/null +++ b/src/prerna/engine/impl/rdbms/migration/SchemaMigrationLock.java @@ -0,0 +1,340 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.engine.impl.rdbms.migration; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.engine.api.IRDBMSEngine; +import prerna.util.ConnectionUtils; +import prerna.util.sql.RdbmsTypeEnum; + +/** + * Cross-node mutex so two nodes in a clustered SEMOSS deployment can't both + * apply the same pending migration to the same external database at once. + * Dialect-dispatched, per the locking research + * ({@code docs/database-migrations/locking-research.md}): + *

    + *
  • Postgres -- a native session-level advisory lock + * ({@code pg_advisory_lock}/{@code pg_advisory_unlock}), keyed by a hash of + * the engine id. Session-level (not transaction-level) is required because + * {@link SchemaMigrationRunner} commits once per migration file, and the + * lock must survive across those per-file commits for the whole batch.
    • + *
  • MySQL / H2 -- no advisory lock support, so a dedicated + * {@code SEMOSS_SCHEMA_LOCK} table (one row per engine's database) is used + * instead, guarded by an atomic {@code INSERT ... WHERE NOT EXISTS}. A lock + * row older than {@link #STALE_LOCK_THRESHOLD_MS} is assumed abandoned by a + * crashed node and is stolen.
    • + *
+ * No Redis dependency -- Redis is an optional deployment feature in SEMOSS + * (only configured when {@code REDIS_ENABLED}/{@code SEMOSS_IS_CLUSTER_REDIS} + * is set), so a Redis-only lock would silently provide no protection in any + * single-node or non-Redis-cluster deployment. See the locking research doc + * for the full comparison; a Redis-based outer guard may be layered on top of + * this in a future iteration but is not required for correctness. + *

+ * Important: for Postgres, the connection used to acquire the lock is + * held open for the lifetime of this object and only released in + * {@link #close()} -- an advisory lock is tied to the session/connection that + * took it, so returning that connection to the pool early would either drop + * the lock or let a different caller unknowingly inherit it. + */ +public final class SchemaMigrationLock implements AutoCloseable { + + private static final Logger classLogger = LogManager.getLogger(SchemaMigrationLock.class); + + /** Public so other migration-package classes (e.g. the OWL sync utility) can exclude this reserved table by name. */ + public static final String LOCK_TABLE = "SEMOSS_SCHEMA_LOCK"; + + /** How long to retry acquiring the lock before giving up. */ + private static final long LOCK_WAIT_MS = 30_000L; + private static final long RETRY_SLEEP_MS = 200L; + /** A MySQL/H2 lock row older than this is assumed abandoned by a crashed node. */ + private static final long STALE_LOCK_THRESHOLD_MS = 600_000L; + + private final IRDBMSEngine engine; + private final RdbmsTypeEnum dbType; + private final long postgresLockKey; + /** Only populated (and kept open) for the Postgres path -- see class Javadoc. */ + private final Connection postgresLockConnection; + + private SchemaMigrationLock(IRDBMSEngine engine, long postgresLockKey, Connection postgresLockConnection) { + this.engine = engine; + this.dbType = engine.getDbType(); + this.postgresLockKey = postgresLockKey; + this.postgresLockConnection = postgresLockConnection; + } + + /** + * @param engine the engine to lock migration execution for + * @return an acquired lock -- release it via try-with-resources + * @throws SchemaMigrationLockTimeoutException if the lock is still held by + * another node/process after + * {@link #LOCK_WAIT_MS} + */ + public static SchemaMigrationLock acquire(IRDBMSEngine engine) { + long lockKey = deriveLockKey(engine.getEngineId()); + if (engine.getDbType() == RdbmsTypeEnum.POSTGRES) { + return acquirePostgresLock(engine, lockKey); + } + return acquireTableLock(engine, lockKey); + } + + @Override + public void close() { + if (dbType == RdbmsTypeEnum.POSTGRES) { + releasePostgresLock(); + } else { + releaseTableLock(engine); + } + } + + // ------------------------------- Postgres -------------------------------- + + private static SchemaMigrationLock acquirePostgresLock(IRDBMSEngine engine, long lockKey) { + long deadline = System.currentTimeMillis() + LOCK_WAIT_MS; + Connection conn = null; + try { + conn = engine.getConnection(); + do { + if (tryPostgresAdvisoryLock(conn, lockKey)) { + classLogger.info( + "Acquired Postgres advisory migration lock for engine '{}' (key={}). To test contention " + + "manually, run 'SELECT pg_try_advisory_lock({});' in a separate psql session " + + "while this lock is held -- it should return false until this engine releases it.", + engine.getEngineId(), lockKey, lockKey); + return new SchemaMigrationLock(engine, lockKey, conn); + } + classLogger.info( + "Migration lock for engine '{}' (key={}) is currently held elsewhere -- retrying.", + engine.getEngineId(), lockKey); + sleepQuietly(); + } while (System.currentTimeMillis() < deadline); + } catch (SQLException e) { + ConnectionUtils.closeAllConnectionsIfPooling(engine, conn); + throw new SchemaMigrationException( + "Failed to acquire Postgres advisory migration lock for engine " + engine.getEngineId(), e); + } + ConnectionUtils.closeAllConnectionsIfPooling(engine, conn); + classLogger.warn("Timed out after {}ms waiting for Postgres advisory migration lock for engine '{}' (key={}).", + LOCK_WAIT_MS, engine.getEngineId(), lockKey); + throw new SchemaMigrationLockTimeoutException(engine.getEngineId(), LOCK_WAIT_MS); + } + + private static boolean tryPostgresAdvisoryLock(Connection conn, long lockKey) throws SQLException { + try (PreparedStatement ps = conn.prepareStatement("SELECT pg_try_advisory_lock(?)")) { + ps.setLong(1, lockKey); + try (ResultSet rs = ps.executeQuery()) { + return rs.next() && rs.getBoolean(1); + } + } + } + + private void releasePostgresLock() { + try (PreparedStatement ps = postgresLockConnection.prepareStatement("SELECT pg_advisory_unlock(?)")) { + ps.setLong(1, postgresLockKey); + ps.execute(); + classLogger.info("Released Postgres advisory migration lock for engine '{}' (key={}).", + engine.getEngineId(), postgresLockKey); + } catch (SQLException e) { + classLogger.error("Failed to release Postgres advisory migration lock for engine '{}'.", + engine.getEngineId(), e); + } finally { + ConnectionUtils.closeAllConnectionsIfPooling(engine, postgresLockConnection); + } + } + + // ------------------------------ MySQL / H2 -------------------------------- + + private static SchemaMigrationLock acquireTableLock(IRDBMSEngine engine, long lockKey) { + ensureLockTable(engine); + long deadline = System.currentTimeMillis() + LOCK_WAIT_MS; + do { + if (tryInsertLockRow(engine)) { + return new SchemaMigrationLock(engine, lockKey, null); + } + if (isLockRowStale(engine)) { + classLogger.warn( + "Migration lock row for engine '{}' is older than {}ms -- assuming the previous holder " + + "crashed and stealing it.", + engine.getEngineId(), STALE_LOCK_THRESHOLD_MS); + deleteLockRow(engine); + continue; + } + sleepQuietly(); + } while (System.currentTimeMillis() < deadline); + throw new SchemaMigrationLockTimeoutException(engine.getEngineId(), LOCK_WAIT_MS); + } + + private static void releaseTableLock(IRDBMSEngine engine) { + deleteLockRow(engine); + } + + private static void ensureLockTable(IRDBMSEngine engine) { + Connection conn = null; + try { + conn = engine.getConnection(); + if (!engine.getQueryUtil().tableExists(conn, LOCK_TABLE, engine.getDatabase(), engine.getSchema())) { + String[] colNames = { "ENGINEID", "LOCKEDBY", "LOCKEDON" }; + String[] types = { "VARCHAR(255)", "VARCHAR(255)", engine.getQueryUtil().getDateWithTimeDataType() }; + String createSql = engine.getQueryUtil().createTable(LOCK_TABLE, colNames, types); + classLogger.info("Creating migration lock table for engine '{}' with sql {}", engine.getEngineId(), + createSql); + engine.insertData(createSql); + } + } catch (Exception e) { + classLogger.error("Failed to ensure migration lock table exists for engine '{}'.", engine.getEngineId(), + e); + throw new SchemaMigrationException("Unable to create or verify " + LOCK_TABLE + " for engine " + + engine.getEngineId(), e); + } finally { + ConnectionUtils.closeAllConnectionsIfPooling(engine, conn); + } + } + + /** + * Atomic guarded insert: only inserts a lock row for this engine if one + * doesn't already exist. There is a narrow race window here since + * {@code SEMOSS_SCHEMA_LOCK} has no unique constraint to fall back on (kept + * consistent with how every other SEMOSS internal table in this feature is + * created -- see the locking research doc's open questions for why a + * unique-constraint retrofit was not chosen). Acceptable for this first + * pass since Postgres (the dialect most likely to run as a real multi-node + * shared external database) uses the race-free advisory lock path instead; + * this fallback mainly protects MySQL, and H2 in practice is never shared + * across nodes at all. + */ + private static boolean tryInsertLockRow(IRDBMSEngine engine) { + Connection conn = null; + try { + conn = engine.getConnection(); + if (lockRowExists(conn, engine.getEngineId())) { + return false; + } + String insertSql = "INSERT INTO " + LOCK_TABLE + " (ENGINEID, LOCKEDBY, LOCKEDON) VALUES (?, ?, ?)"; + try (PreparedStatement ps = conn.prepareStatement(insertSql)) { + ps.setString(1, engine.getEngineId()); + ps.setString(2, MigrationHistoryRecord.SYSTEM_APPLIED_BY); + ps.setTimestamp(3, new Timestamp(System.currentTimeMillis())); + ps.execute(); + if (!conn.getAutoCommit()) { + conn.commit(); + } + } + return true; + } catch (SQLException e) { + classLogger.warn("Could not insert migration lock row for engine '{}' -- likely already held.", + engine.getEngineId(), e); + return false; + } finally { + ConnectionUtils.closeAllConnectionsIfPooling(engine, conn); + } + } + + private static boolean lockRowExists(Connection conn, String engineId) throws SQLException { + String selectSql = "SELECT LOCKEDON FROM " + LOCK_TABLE + " WHERE ENGINEID = ?"; + try (PreparedStatement ps = conn.prepareStatement(selectSql)) { + ps.setString(1, engineId); + try (ResultSet rs = ps.executeQuery()) { + return rs.next(); + } + } + } + + private static boolean isLockRowStale(IRDBMSEngine engine) { + Connection conn = null; + String selectSql = "SELECT LOCKEDON FROM " + LOCK_TABLE + " WHERE ENGINEID = ?"; + try { + conn = engine.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(selectSql)) { + ps.setString(1, engine.getEngineId()); + try (ResultSet rs = ps.executeQuery()) { + if (!rs.next()) { + // no row at all -- nothing to steal, but not "stale" either + return false; + } + Timestamp lockedOn = rs.getTimestamp("LOCKEDON"); + return lockedOn != null + && System.currentTimeMillis() - lockedOn.getTime() > STALE_LOCK_THRESHOLD_MS; + } + } + } catch (SQLException e) { + classLogger.error("Failed to check migration lock staleness for engine '{}'.", engine.getEngineId(), e); + return false; + } finally { + ConnectionUtils.closeAllConnectionsIfPooling(engine, conn); + } + } + + private static void deleteLockRow(IRDBMSEngine engine) { + Connection conn = null; + String deleteSql = "DELETE FROM " + LOCK_TABLE + " WHERE ENGINEID = ?"; + try { + conn = engine.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(deleteSql)) { + ps.setString(1, engine.getEngineId()); + ps.execute(); + if (!conn.getAutoCommit()) { + conn.commit(); + } + } + } catch (SQLException e) { + classLogger.error("Failed to release migration lock row for engine '{}'.", engine.getEngineId(), e); + } finally { + ConnectionUtils.closeAllConnectionsIfPooling(engine, conn); + } + } + + // -------------------------------- shared ---------------------------------- + + private static void sleepQuietly() { + try { + Thread.sleep(RETRY_SLEEP_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + + /** + * Derives a stable advisory-lock key from an engine id. Collisions between + * two different engine ids are theoretically possible (32-bit hash space) + * but negligible in practice for UUID-based engine ids, and would only + * cause unrelated engines to needlessly serialize on the same lock, never + * an incorrect migration outcome. + */ + static long deriveLockKey(String engineId) { + return ((long) ("semoss_migration:" + engineId).hashCode()) & 0x7fffffffffffffffL; + } + +} diff --git a/src/prerna/engine/impl/rdbms/migration/SchemaMigrationLockTimeoutException.java b/src/prerna/engine/impl/rdbms/migration/SchemaMigrationLockTimeoutException.java new file mode 100644 index 00000000000..0ad85470268 --- /dev/null +++ b/src/prerna/engine/impl/rdbms/migration/SchemaMigrationLockTimeoutException.java @@ -0,0 +1,46 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.engine.impl.rdbms.migration; + +/** + * Thrown when {@link SchemaMigrationLock#acquire} cannot obtain the + * cross-node migration lock within its timeout. A subtype of + * {@link SchemaMigrationException} so existing callers that only catch the + * parent type still handle it, while {@link SchemaMigrationRunner} catches + * this subtype specifically to re-check migration history before deciding + * whether it's actually safe to proceed (see its Javadoc). + */ +public class SchemaMigrationLockTimeoutException extends SchemaMigrationException { + + private static final long serialVersionUID = 1L; + + public SchemaMigrationLockTimeoutException(String engineId, long waitedMs) { + super("Timed out after " + waitedMs + "ms waiting for the migration lock on engine " + engineId); + } + +} diff --git a/src/prerna/engine/impl/rdbms/migration/SchemaMigrationRunner.java b/src/prerna/engine/impl/rdbms/migration/SchemaMigrationRunner.java new file mode 100644 index 00000000000..ff9ece9f497 --- /dev/null +++ b/src/prerna/engine/impl/rdbms/migration/SchemaMigrationRunner.java @@ -0,0 +1,277 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.engine.impl.rdbms.migration; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.engine.api.IRDBMSEngine; +import prerna.util.ConnectionUtils; + +/** + * Entry point called from {@code IRDBMSEngine.open(Properties)} when + * {@code ENABLE_MIGRATIONS=true} is set on the engine's smss. Discovers + * pending {@code V__.sql} files under the engine's own + * {@code assets/.migrations} folder, runs each one in version order, records + * the outcome to {@code SEMOSS_SCHEMA_HISTORY} (inside the target engine's + * own database), and re-syncs the OWL metamodel once per file that actually + * ran. + *

+ * Any failure -- a failing SQL statement, an out-of-order version, or a + * checksum mismatch against an already-applied version -- throws + * {@link SchemaMigrationException}, which is left to propagate out of + * {@code open()}. Confirmed in {@code Utility.loadEngine()} that a thrown + * {@code open()} exception means the engine is never registered in + * {@code DIHelper} -- so a failed migration leaves the engine fully unusable + * rather than partially available with a half-migrated schema. + *

+ * Concurrency across a single JVM is already handled upstream -- + * {@code Utility.getDatabase()}/{@code baseGetEngine()} take a per-engine + * {@code ReentrantLock} (via {@code EngineSyncUtility.getEngineLock}) before + * calling {@code open()}. Cross-node locking in a clustered deployment is + * handled by {@link SchemaMigrationLock} -- a real Postgres advisory lock, or + * a lock-table fallback for MySQL/H2 -- wrapping the entire method below (see + * {@code docs/database-migrations/locking-research.md} for the full + * rationale). + */ +public final class SchemaMigrationRunner { + + private static final Logger classLogger = LogManager.getLogger(SchemaMigrationRunner.class); + + private SchemaMigrationRunner() { + // utility class + } + + /** + * @param engine the engine to run pending migrations against -- + * must already have a live connection (called after + * {@code this.engineConnected = true} in + * {@code RDBMSNativeEngine.open()}) + * @param migrationsFolder the engine's {@code assets/.migrations} folder + */ + public static void runPendingMigrations(IRDBMSEngine engine, File migrationsFolder) { + try (SchemaMigrationLock lock = SchemaMigrationLock.acquire(engine)) { + runPendingMigrationsLocked(engine, migrationsFolder); + } catch (SchemaMigrationLockTimeoutException timeout) { + handleLockTimeout(engine, migrationsFolder, timeout); + } + } + + /** + * Called after a lock-acquisition timeout. Most likely explanation: another + * node already holds the lock and is (or just finished) applying the same + * pending migrations -- in which case there's nothing left for this node to + * do and {@code open()} can proceed normally. Only fails {@code open()} if + * migrations are genuinely still pending/failed after the timeout, which + * means either contention is unusually long-lived or the previous holder + * crashed mid-migration and left a real failure behind. + */ + private static void handleLockTimeout(IRDBMSEngine engine, File migrationsFolder, + SchemaMigrationLockTimeoutException timeout) { + List allMigrations = MigrationFileUtils.scanMigrationsFolder(migrationsFolder); + List history = MigrationHistoryUtils.getHistory(engine); + Set appliedVersions = new HashSet<>(); + for (MigrationHistoryRecord record : history) { + if (record.isSuccess()) { + appliedVersions.add(record.getVersion()); + } + } + + boolean stillPending = allMigrations.stream().anyMatch(m -> !appliedVersions.contains(m.getVersion())); + if (!stillPending) { + classLogger.info( + "Migration lock for engine '{}' timed out, but all migrations were already applied by " + + "another node -- proceeding.", + engine.getEngineId()); + return; + } + throw new SchemaMigrationException( + "Could not acquire the migration lock for engine " + engine.getEngineId() + + " and migrations are still pending. Another node may still be migrating, or a previous " + + "run crashed mid-migration -- check " + "SEMOSS_SCHEMA_HISTORY for details.", + timeout); + } + + private static void runPendingMigrationsLocked(IRDBMSEngine engine, File migrationsFolder) { + ensureMigrationsFolder(migrationsFolder, engine.getEngineId()); + MigrationHistoryUtils.ensureHistoryTable(engine); + + List allMigrations = MigrationFileUtils.scanMigrationsFolder(migrationsFolder); + if (allMigrations.isEmpty()) { + return; + } + + List history = MigrationHistoryUtils.getHistory(engine); + Set appliedVersions = new HashSet<>(); + String highestAppliedVersion = null; + for (MigrationHistoryRecord record : history) { + if (record.isSuccess()) { + appliedVersions.add(record.getVersion()); + if (highestAppliedVersion == null + || MigrationFileUtils.compareVersions(record.getVersion(), highestAppliedVersion) > 0) { + highestAppliedVersion = record.getVersion(); + } + } + } + + boolean ranAtLeastOne = false; + for (MigrationFile migration : allMigrations) { + if (appliedVersions.contains(migration.getVersion())) { + verifyChecksumUnchanged(migration, history); + continue; + } + + rejectIfOutOfOrder(migration, highestAppliedVersion); + runMigration(engine, migration); + ranAtLeastOne = true; + // sync once per file -- keeps OWL correct incrementally in case a later + // file in this same batch fails + RdbmsMigrationOwlSyncUtils.syncOwlAfterMigration(engine); + } + + if (ranAtLeastOne) { + classLogger.info("Completed pending migrations for engine '{}'.", engine.getEngineId()); + } + } + + /** + * Ensures the engine's {@code assets/.migrations} folder exists the moment + * {@code ENABLE_MIGRATIONS} is turned on -- not lazily, only whenever + * someone happens to save a migration through the UI. An admin (or a + * future external tool) should be able to find the folder in place as soon + * as the flag is set, even before a single migration has ever been + * created. + */ + private static void ensureMigrationsFolder(File migrationsFolder, String engineId) { + try { + Files.createDirectories(migrationsFolder.toPath()); + } catch (IOException e) { + classLogger.error("Failed to create migrations folder '{}' for engine '{}'.", + migrationsFolder.getAbsolutePath(), engineId, e); + throw new SchemaMigrationException( + "Unable to create migrations folder for engine " + engineId, e); + } + } + + private static void verifyChecksumUnchanged(MigrationFile migration, List history) { + String currentChecksum = MigrationFileUtils.computeChecksum(migration.getSqlContent()); + for (MigrationHistoryRecord record : history) { + if (record.isSuccess() && record.getVersion().equals(migration.getVersion()) + && !record.getChecksum().equals(currentChecksum)) { + throw new SchemaMigrationException("Migration " + migration.getFileName() + + " has already been applied but its content has changed since then (checksum mismatch). " + + "Restore the original file content or create a new version instead of editing it."); + } + } + } + + private static void rejectIfOutOfOrder(MigrationFile migration, String highestAppliedVersion) { + if (highestAppliedVersion != null + && MigrationFileUtils.compareVersions(migration.getVersion(), highestAppliedVersion) < 0) { + throw new SchemaMigrationException("Migration " + migration.getFileName() + " has version " + + migration.getVersion() + ", which is lower than the highest already-applied version " + + highestAppliedVersion + ". Out-of-order migrations are not supported."); + } + } + + private static void runMigration(IRDBMSEngine engine, MigrationFile migration) { + long start = System.currentTimeMillis(); + String checksum = MigrationFileUtils.computeChecksum(migration.getSqlContent()); + boolean success = false; + String failureReason = null; + Connection conn = null; + try { + engine.setAutoCommit(false); + conn = engine.getConnection(); + for (String statement : MigrationFileUtils.splitStatements(migration.getSqlContent())) { + try (PreparedStatement ps = conn.prepareStatement(statement)) { + ps.execute(); + } + } + long executionTimeMs = System.currentTimeMillis() - start; + MigrationHistoryRecord record = new MigrationHistoryRecord(migration.getVersion(), migration.getFileName(), + checksum, MigrationHistoryRecord.SYSTEM_APPLIED_BY, new Timestamp(System.currentTimeMillis()), + executionTimeMs, true, null); + // Insert on the SAME connection/transaction as the migration's own SQL so + // both commit -- or both roll back -- together. Confirmed from Flyway's own + // source (SchemaHistory.java: "a migration failure automatically triggers a + // rollback of all changes, including the ones in the schema history table") + // that this pairing is the real, correct behavior -- recording success on a + // separate connection/transaction (the earlier version of this method) left a + // durability gap: a crash between the two commits would leave the migration + // permanently applied with no history row, causing it to look "pending" and + // be retried against non-idempotent DDL next time. + MigrationHistoryUtils.insertHistoryRow(conn, record); + engine.commit(); + success = true; + } catch (Exception e) { + classLogger.error("Migration '{}' failed against engine '{}'.", migration.getFileName(), + engine.getEngineId(), e); + failureReason = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); + rollbackQuietly(conn, migration); + } finally { + ConnectionUtils.closeAllConnectionsIfPooling(engine, conn); + } + + if (!success) { + // the migration's own transaction (and any history row attempted inside it) + // was just rolled back -- record the failure on its own fresh connection so + // it's actually durable and visible in the Migrations tab + long executionTimeMs = System.currentTimeMillis() - start; + MigrationHistoryRecord failureRecord = new MigrationHistoryRecord(migration.getVersion(), + migration.getFileName(), checksum, MigrationHistoryRecord.SYSTEM_APPLIED_BY, + new Timestamp(System.currentTimeMillis()), executionTimeMs, false, failureReason); + MigrationHistoryUtils.recordMigration(engine, failureRecord); + throw new SchemaMigrationException( + "Migration " + migration.getFileName() + " failed: " + failureReason); + } + } + + private static void rollbackQuietly(Connection conn, MigrationFile migration) { + if (conn == null) { + return; + } + try { + conn.rollback(); + } catch (SQLException rollbackEx) { + classLogger.error("Failed to roll back migration '{}'.", migration.getFileName(), rollbackEx); + } + } + +} diff --git a/src/prerna/reactor/database/migration/GetEngineMigrationsEnabledReactor.java b/src/prerna/reactor/database/migration/GetEngineMigrationsEnabledReactor.java new file mode 100644 index 00000000000..98b8306cf04 --- /dev/null +++ b/src/prerna/reactor/database/migration/GetEngineMigrationsEnabledReactor.java @@ -0,0 +1,89 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.database.migration; + +import java.util.Properties; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityQueryUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Constants; +import prerna.util.DIHelper; +import prerna.util.Utility; + +/** + * Reports whether {@code ENABLE_MIGRATIONS} is set on an engine's smss -- + * used by the Migrations tab to decide whether to show itself at all, + * without requiring the OWNER-only access {@code GetEngineSMSS} needs (this + * only reveals one boolean, not the full smss content, so a view-level check + * is sufficient). + * + * 

GetEngineMigrationsEnabled(engine = ["<engineId>"]);
+ * + * Returns: BOOLEAN. + */ +public class GetEngineMigrationsEnabledReactor extends AbstractReactor { + + public GetEngineMigrationsEnabledReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.ENGINE.getKey() }; + this.keyRequired = new int[] { 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String rawEngineId = this.keyValue.get(this.keysToGet[0]); + if (rawEngineId == null || rawEngineId.trim().isEmpty()) { + throw new IllegalArgumentException("Must provide an engine id to check"); + } + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("User must be logged in"); + } + String engineId = SecurityQueryUtils.testUserEngineIdForAlias(user, rawEngineId); + if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { + throw new IllegalArgumentException( + "Engine " + engineId + " does not exist or user does not have access to view it"); + } + + String smssFile = (String) DIHelper.getInstance().getEngineProperty(engineId + "_" + Constants.STORE); + boolean enabled = false; + if (smssFile != null) { + Properties prop = Utility.loadProperties(smssFile); + enabled = Boolean.parseBoolean(prop.getProperty(Constants.ENABLE_MIGRATIONS)); + } + + return new NounMetadata(enabled, PixelDataType.BOOLEAN); + } + +} diff --git a/src/prerna/reactor/database/migration/ListEngineMigrationsReactor.java b/src/prerna/reactor/database/migration/ListEngineMigrationsReactor.java new file mode 100644 index 00000000000..338ff72a6e8 --- /dev/null +++ b/src/prerna/reactor/database/migration/ListEngineMigrationsReactor.java @@ -0,0 +1,116 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.database.migration; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityQueryUtils; +import prerna.engine.api.IDatabaseEngine; +import prerna.engine.api.IRDBMSEngine; +import prerna.engine.impl.rdbms.migration.MigrationFileUtils; +import prerna.engine.impl.rdbms.migration.MigrationStatus; +import prerna.engine.impl.rdbms.migration.MigrationStatusUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Read-only status board backing the Migrations tab -- merges the + * {@code V__.sql} files on disk with + * {@code SEMOSS_SCHEMA_HISTORY} run outcomes into one row per version, the + * same merge Flyway's {@code info} command performs. Does not run, create, or + * modify any migration; {@code SchemaMigrationRunner} (called automatically + * from {@code IRDBMSEngine.open()}) is the only thing that does that. + * + * 
ListEngineMigrations(engine = ["<engineId>"]);
+ * + * Returns: VECTOR of maps, one per version, ordered ascending, each + * containing {@code version}, {@code description}, {@code fileName}, + * {@code state} (PENDING / SUCCESS / FAILED / MISSING / OUTDATED), + * {@code appliedBy}, {@code appliedOn}, {@code executionTimeMs}, and + * {@code errorMessage}. + */ +public class ListEngineMigrationsReactor extends AbstractReactor { + + public ListEngineMigrationsReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.ENGINE.getKey() }; + this.keyRequired = new int[] { 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String rawEngineId = this.keyValue.get(this.keysToGet[0]); + if (rawEngineId == null || rawEngineId.trim().isEmpty()) { + throw new IllegalArgumentException("Must provide an engine id to list migrations for"); + } + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("User must be logged in"); + } + String engineId = SecurityQueryUtils.testUserEngineIdForAlias(user, rawEngineId); + if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { + throw new IllegalArgumentException( + "Engine " + engineId + " does not exist or user does not have access to view it"); + } + + IDatabaseEngine database = Utility.getDatabase(engineId); + if (!(database instanceof IRDBMSEngine rdbmsEngine)) { + throw new IllegalArgumentException("Engine " + engineId + " is not a JDBC database engine"); + } + + File migrationsFolder = MigrationFileUtils.getMigrationsFolder(rdbmsEngine); + List statuses = MigrationStatusUtils.getStatus(rdbmsEngine, migrationsFolder); + + List> rows = new ArrayList<>(); + for (MigrationStatus status : statuses) { + Map row = new HashMap<>(); + row.put("version", status.getVersion()); + row.put("description", status.getDescription()); + row.put("fileName", status.getFileName()); + row.put("state", status.getState().name()); + row.put("appliedBy", status.getAppliedBy()); + row.put("appliedOn", status.getAppliedOn()); + row.put("executionTimeMs", status.getExecutionTimeMs()); + row.put("errorMessage", status.getErrorMessage()); + rows.add(row); + } + + return new NounMetadata(rows, PixelDataType.VECTOR); + } + +} diff --git a/src/prerna/reactor/database/migration/SaveEngineMigrationReactor.java b/src/prerna/reactor/database/migration/SaveEngineMigrationReactor.java new file mode 100644 index 00000000000..31bafbbe9cc --- /dev/null +++ b/src/prerna/reactor/database/migration/SaveEngineMigrationReactor.java @@ -0,0 +1,195 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.database.migration; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityQueryUtils; +import prerna.engine.api.IDatabaseEngine; +import prerna.engine.api.IRDBMSEngine; +import prerna.engine.impl.rdbms.migration.MigrationFile; +import prerna.engine.impl.rdbms.migration.MigrationFileUtils; +import prerna.engine.impl.rdbms.migration.MigrationStatus; +import prerna.engine.impl.rdbms.migration.MigrationStatusUtils; +import prerna.engine.impl.rdbms.migration.SchemaMigrationException; +import prerna.engine.impl.rdbms.migration.SchemaMigrationRunner; +import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Creates the next {@code V__.sql} file under an + * engine's own {@code assets/.migrations} folder -- and only this reactor + * (via the Migrations tab) does that. The folder/file are never meant to be + * hand-created on disk; this is the missing piece that makes that true. + * Immediately runs the newly-created migration afterward (reusing + * {@link SchemaMigrationRunner} -- the same lock/checksum/OWL-sync pipeline + * {@code IRDBMSEngine.open()} uses), so the effect of a saved migration is + * visible right away instead of requiring a full engine reload. + * + * 
+ * SaveEngineMigration(engine = ["<engineId>"], sql = ["ALTER TABLE ...;"],
+ *     description = ["add_status_column"]);
+ *
+ * + * Returns: MAP containing {@code version}, {@code fileName}, {@code success}, + * and {@code errorMessage} (null on success). + */ +public class SaveEngineMigrationReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(SaveEngineMigrationReactor.class); + + public SaveEngineMigrationReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.ENGINE.getKey(), ReactorKeysEnum.SQL.getKey(), + ReactorKeysEnum.DESCRIPTION.getKey() }; + this.keyRequired = new int[] { 1, 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String rawEngineId = this.keyValue.get(this.keysToGet[0]); + String sqlContent = this.keyValue.get(this.keysToGet[1]); + String description = this.keyValue.get(this.keysToGet[2]); + if (rawEngineId == null || rawEngineId.trim().isEmpty()) { + throw new IllegalArgumentException("Must provide an engine id to save a migration against"); + } + if (sqlContent == null || sqlContent.trim().isEmpty()) { + throw new IllegalArgumentException("Must provide SQL content for the migration"); + } + if (description == null || description.trim().isEmpty()) { + throw new IllegalArgumentException("Must provide a short description for the migration"); + } + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("User must be logged in"); + } + String engineId = SecurityQueryUtils.testUserEngineIdForAlias(user, rawEngineId); + if (!SecurityEngineUtils.userCanEditEngine(user, engineId)) { + throw new IllegalArgumentException( + "Engine " + engineId + " does not exist or user does not have access to edit it"); + } + + IDatabaseEngine database = Utility.getDatabase(engineId); + if (!(database instanceof IRDBMSEngine rdbmsEngine)) { + throw new IllegalArgumentException("Engine " + engineId + " is not a JDBC database engine"); + } + + File migrationsFolder = MigrationFileUtils.getMigrationsFolder(rdbmsEngine); + String version = writeMigrationFile(migrationsFolder, sqlContent, description); + + Map response = new HashMap<>(); + response.put("version", version); + try { + SchemaMigrationRunner.runPendingMigrations(rdbmsEngine, migrationsFolder); + response.put("success", true); + response.put("errorMessage", null); + } catch (SchemaMigrationException e) { + classLogger.error("Saved migration version '{}' for engine '{}' failed to run.", version, engineId, e); + response.put("success", false); + response.put("errorMessage", latestErrorMessage(rdbmsEngine, migrationsFolder, version, e)); + } + return new NounMetadata(response, PixelDataType.MAP); + } + + /** + * Scans the folder for the current highest version, writes the new file as + * the next one, and creates the folder itself if this is the engine's first + * migration -- the folder/file creation this reactor exists to own instead + * of a person doing it by hand on disk. + */ + private String writeMigrationFile(File migrationsFolder, String sqlContent, String description) { + try { + Files.createDirectories(migrationsFolder.toPath()); + List existing = MigrationFileUtils.scanMigrationsFolder(migrationsFolder); + String version = nextVersion(existing); + String fileName = "V" + version + "__" + sanitizeDescription(description) + ".sql"; + Files.writeString(migrationsFolder.toPath().resolve(fileName), sqlContent, StandardCharsets.UTF_8); + return version; + } catch (IOException e) { + classLogger.error("Failed to write migration file under '{}'.", migrationsFolder.getAbsolutePath(), e); + throw new SchemaMigrationException( + "Unable to write migration file under " + migrationsFolder.getAbsolutePath(), e); + } + } + + private String nextVersion(List existing) { + if (existing.isEmpty()) { + return "1"; + } + String highest = existing.stream().map(MigrationFile::getVersion) + .max(MigrationFileUtils::compareVersions).orElse("0"); + int majorSegment = Integer.parseInt(highest.split("\\.")[0]); + return String.valueOf(majorSegment + 1); + } + + private String sanitizeDescription(String description) { + String sanitized = description.trim().replaceAll("[^a-zA-Z0-9-_]", "_"); + return sanitized.isEmpty() ? "migration" : sanitized; + } + + /** + * The runner already recorded the failure to {@code SEMOSS_SCHEMA_HISTORY}; + * surface that recorded reason if available since it's the most accurate + * (e.g. an out-of-order/checksum rejection never even reaches SQL + * execution, so it never gets a history row -- in that case fall back to + * the exception's own message). + */ + private String latestErrorMessage(IRDBMSEngine engine, File migrationsFolder, String version, + SchemaMigrationException fallback) { + List statuses = MigrationStatusUtils.getStatus(engine, migrationsFolder); + return statuses.stream().filter(s -> s.getVersion().equals(version)).findFirst() + .map(MigrationStatus::getErrorMessage).orElseGet(() -> fallback.getMessage()); + } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + // this creates AND immediately runs arbitrary DDL/DML -- must never be + // agent-auto-triggerable + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + return meta; + } + +} diff --git a/src/prerna/util/Constants.java b/src/prerna/util/Constants.java index b5dae95ade3..b6acefc2df9 100644 --- a/src/prerna/util/Constants.java +++ b/src/prerna/util/Constants.java @@ -654,6 +654,9 @@ public class Constants { public static final String USE_OUTER_JOINS = "USE_OUTER_JOINS";// if present and true use outer joins instead of // inner joins public static final String USE_CONNECTION_POOLING = "USE_CONNECTION_POOLING"; + // if present and true, run pending SQL migrations from the engine's own + // assets/migrations folder on every open() + public static final String ENABLE_MIGRATIONS = "ENABLE_MIGRATIONS"; public static final String H2_BASE_CONNECTION_URL = "jdbc:h2:@" + Constants.BASE_FOLDER + "@" + System.getProperty("file.separator") + "@ENGINE" + Constants.ENGINE + "@" + System.getProperty("file.separator") diff --git a/src/prerna/util/FileSystemUtil.java b/src/prerna/util/FileSystemUtil.java index 1f0fce0106a..01f0121b529 100644 --- a/src/prerna/util/FileSystemUtil.java +++ b/src/prerna/util/FileSystemUtil.java @@ -223,7 +223,7 @@ public static void searchRecursive(File dir, Pattern pattern, int baseLen, List< for (File f : entries) { String name = f.getName(); - // hide .git directory and .admin directory + // hide .git, .admin, and .migrations directories if (isHiddenAsset(f)) { continue; } @@ -270,10 +270,14 @@ private static Map createMeta(File f, String relativePath, boole /** * Determine whether an asset should be hidden from file explorer listings. - * Hides the ".git" directory and the ".admin" directory. Only the leaf name is - * inspected, so this is suitable for filtering the direct children of an - * already-validated directory; to also reject entries that live inside - * a hidden directory, use {@link #isWithinHiddenAsset(File, int)}. + * Hides the ".git" directory, the ".admin" directory, and the ".migrations" + * directory (an engine's own SQL migration files -- system-managed, not + * meant to be browsed/edited by hand; see + * {@code prerna.engine.impl.rdbms.migration.MigrationFileUtils}). Only the + * leaf name is inspected, so this is suitable for filtering the direct + * children of an already-validated directory; to also reject entries that + * live inside a hidden directory, use + * {@link #isWithinHiddenAsset(File, int)}. * * @param f the file or directory being considered * @return true if the entry should be excluded from the results @@ -284,10 +288,11 @@ private static boolean isHiddenAsset(File f) { /** * Determine whether a file or directory sits at or beneath a hidden asset - * (".git" or ".admin") by inspecting every segment of its assets-relative path. - * Unlike {@link #isHiddenAsset(File)}, which only looks at the leaf name, this - * also blocks access when an ancestor segment is hidden — e.g. a - * caller targeting ".git/hooks" or ".admin/secrets" as the starting point of a + * (".git", ".admin", or ".migrations") by inspecting every segment of its + * assets-relative path. Unlike {@link #isHiddenAsset(File)}, which only + * looks at the leaf name, this also blocks access when an ancestor + * segment is hidden — e.g. a caller targeting ".git/hooks", + * ".admin/secrets", or ".migrations/V1__init.sql" as the starting point of a * browse or search. * * @param f the file or directory being considered @@ -308,11 +313,11 @@ private static boolean isWithinHiddenAsset(File f, int baseLen) { /** * @param name a single path-segment name - * @return true if the name is one of the hidden asset directories (".git" or - * ".admin") + * @return true if the name is one of the hidden asset directories (".git", + * ".admin", or ".migrations") */ private static boolean isHiddenName(String name) { - return name.equals(".git") || name.equals(".admin"); + return name.equals(".git") || name.equals(".admin") || name.equals(".migrations"); } /** From d5f68bad90afc24c0acd4cb378dc4e1e05bbd786 Mon Sep 17 00:00:00 2001 From: "Patel, Parth" Date: Fri, 31 Jul 2026 08:50:16 -0400 Subject: [PATCH 2/3] chore: fixes for bad migrations --- .../rdbms/migration/SchemaMigrationLock.java | 33 ++-- .../migration/SchemaMigrationRunner.java | 48 +++++- .../migration/SaveEngineMigrationReactor.java | 66 +++++++- .../SchemaMigrationRunnerUnitTests.java | 124 +++++++++++++++ .../SaveEngineMigrationReactorUnitTests.java | 144 ++++++++++++++++++ 5 files changed, 396 insertions(+), 19 deletions(-) create mode 100644 test/prerna/engine/impl/migration/SchemaMigrationRunnerUnitTests.java create mode 100644 test/prerna/reactor/database/migration/SaveEngineMigrationReactorUnitTests.java diff --git a/src/prerna/engine/impl/rdbms/migration/SchemaMigrationLock.java b/src/prerna/engine/impl/rdbms/migration/SchemaMigrationLock.java index 860d9948f0d..8b279a7282e 100644 --- a/src/prerna/engine/impl/rdbms/migration/SchemaMigrationLock.java +++ b/src/prerna/engine/impl/rdbms/migration/SchemaMigrationLock.java @@ -27,6 +27,7 @@ *******************************************************************************/ package prerna.engine.impl.rdbms.migration; +import java.lang.management.ManagementFactory; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; @@ -82,6 +83,11 @@ public final class SchemaMigrationLock implements AutoCloseable { private static final long RETRY_SLEEP_MS = 200L; /** A MySQL/H2 lock row older than this is assumed abandoned by a crashed node. */ private static final long STALE_LOCK_THRESHOLD_MS = 600_000L; + /** + * Identifies which process/node holds a lock row, for stale-lock diagnosis -- + * {@code @} via the JDK's own runtime bean, no new dependency. + */ + private static final String LOCK_OWNER_ID = ManagementFactory.getRuntimeMXBean().getName(); private final IRDBMSEngine engine; private final RdbmsTypeEnum dbType; @@ -208,7 +214,13 @@ private static void ensureLockTable(IRDBMSEngine engine) { if (!engine.getQueryUtil().tableExists(conn, LOCK_TABLE, engine.getDatabase(), engine.getSchema())) { String[] colNames = { "ENGINEID", "LOCKEDBY", "LOCKEDON" }; String[] types = { "VARCHAR(255)", "VARCHAR(255)", engine.getQueryUtil().getDateWithTimeDataType() }; - String createSql = engine.getQueryUtil().createTable(LOCK_TABLE, colNames, types); + // PRIMARY KEY on ENGINEID makes the insert-guarded-by-select in + // tryInsertLockRow() race-free: two nodes racing to insert the same + // ENGINEID can no longer both succeed -- the loser gets a constraint + // violation and reports "not acquired" instead of silently double-locking. + Object[] customConstraints = { "PRIMARY KEY", null, null }; + String createSql = engine.getQueryUtil().createTableWithCustomConstraints(LOCK_TABLE, colNames, types, + customConstraints); classLogger.info("Creating migration lock table for engine '{}' with sql {}", engine.getEngineId(), createSql); engine.insertData(createSql); @@ -224,16 +236,13 @@ private static void ensureLockTable(IRDBMSEngine engine) { } /** - * Atomic guarded insert: only inserts a lock row for this engine if one - * doesn't already exist. There is a narrow race window here since - * {@code SEMOSS_SCHEMA_LOCK} has no unique constraint to fall back on (kept - * consistent with how every other SEMOSS internal table in this feature is - * created -- see the locking research doc's open questions for why a - * unique-constraint retrofit was not chosen). Acceptable for this first - * pass since Postgres (the dialect most likely to run as a real multi-node - * shared external database) uses the race-free advisory lock path instead; - * this fallback mainly protects MySQL, and H2 in practice is never shared - * across nodes at all. + * Guarded insert: the pre-check ({@code lockRowExists}) is only a fast-path + * short-circuit to avoid a wasted round trip when contention is likely -- + * the actual race-freedom comes from the {@code PRIMARY KEY} on + * {@code ENGINEID} (see {@code ensureLockTable}). If two nodes both pass the + * pre-check at the same instant, only one {@code INSERT} can succeed; the + * loser gets a primary-key-violation {@link SQLException} and correctly + * reports "not acquired" via the existing catch block below. */ private static boolean tryInsertLockRow(IRDBMSEngine engine) { Connection conn = null; @@ -245,7 +254,7 @@ private static boolean tryInsertLockRow(IRDBMSEngine engine) { String insertSql = "INSERT INTO " + LOCK_TABLE + " (ENGINEID, LOCKEDBY, LOCKEDON) VALUES (?, ?, ?)"; try (PreparedStatement ps = conn.prepareStatement(insertSql)) { ps.setString(1, engine.getEngineId()); - ps.setString(2, MigrationHistoryRecord.SYSTEM_APPLIED_BY); + ps.setString(2, LOCK_OWNER_ID); ps.setTimestamp(3, new Timestamp(System.currentTimeMillis())); ps.execute(); if (!conn.getAutoCommit()) { diff --git a/src/prerna/engine/impl/rdbms/migration/SchemaMigrationRunner.java b/src/prerna/engine/impl/rdbms/migration/SchemaMigrationRunner.java index ff9ece9f497..9840ca11b83 100644 --- a/src/prerna/engine/impl/rdbms/migration/SchemaMigrationRunner.java +++ b/src/prerna/engine/impl/rdbms/migration/SchemaMigrationRunner.java @@ -158,6 +158,7 @@ private static void runPendingMigrationsLocked(IRDBMSEngine engine, File migrati } rejectIfOutOfOrder(migration, highestAppliedVersion); + rejectIfPreviouslyFailedUnchanged(migration, history); runMigration(engine, migration); ranAtLeastOne = true; // sync once per file -- keeps OWL correct incrementally in case a later @@ -201,6 +202,41 @@ private static void verifyChecksumUnchanged(MigrationFile migration, List + * Editing the file changes its checksum, which naturally exits this + * short-circuit and treats it as a fresh attempt -- the same "fix the + * content to retry" mechanism {@link #verifyChecksumUnchanged} already + * relies on for the applied-version case, just applied to the failed + * case instead. + */ + // package-private (not private) so it's directly unit-testable without a + // real database, same convention as SchemaMigrationLock.deriveLockKey + static void rejectIfPreviouslyFailedUnchanged(MigrationFile migration, List history) { + String currentChecksum = MigrationFileUtils.computeChecksum(migration.getSqlContent()); + for (MigrationHistoryRecord record : history) { + if (!record.isSuccess() && record.getVersion().equals(migration.getVersion()) + && record.getChecksum().equals(currentChecksum)) { + throw new SchemaMigrationException("Migration " + migration.getFileName() + + " already failed with this exact content on " + record.getAppliedOn() + ": " + + record.getDescription() + ". Fix the SQL -- which changes its checksum -- to retry, " + + "or check SEMOSS_SCHEMA_HISTORY for full details."); + } + } + } + private static void rejectIfOutOfOrder(MigrationFile migration, String highestAppliedVersion) { if (highestAppliedVersion != null && MigrationFileUtils.compareVersions(migration.getVersion(), highestAppliedVersion) < 0) { @@ -217,8 +253,16 @@ private static void runMigration(IRDBMSEngine engine, MigrationFile migration) { String failureReason = null; Connection conn = null; try { - engine.setAutoCommit(false); + // Set/commit autocommit on the specific borrowed connection, never via + // engine.setAutoCommit()/engine.commit() -- those mutate persistent, + // engine-wide state (and engine.commit() is a documented no-op under + // connection pooling), which would either leak autocommit=false onto + // every future connection from this engine or silently never commit at + // all. HikariCP resets a leased connection's autoCommit/isolation back to + // the pool default when it's returned via ConnectionUtils, so scoping this + // to `conn` is also safe to leave uncleaned-up on close. conn = engine.getConnection(); + conn.setAutoCommit(false); for (String statement : MigrationFileUtils.splitStatements(migration.getSqlContent())) { try (PreparedStatement ps = conn.prepareStatement(statement)) { ps.execute(); @@ -238,7 +282,7 @@ checksum, MigrationHistoryRecord.SYSTEM_APPLIED_BY, new Timestamp(System.current // permanently applied with no history row, causing it to look "pending" and // be retried against non-idempotent DDL next time. MigrationHistoryUtils.insertHistoryRow(conn, record); - engine.commit(); + conn.commit(); success = true; } catch (Exception e) { classLogger.error("Migration '{}' failed against engine '{}'.", migration.getFileName(), diff --git a/src/prerna/reactor/database/migration/SaveEngineMigrationReactor.java b/src/prerna/reactor/database/migration/SaveEngineMigrationReactor.java index 31bafbbe9cc..423dbf99234 100644 --- a/src/prerna/reactor/database/migration/SaveEngineMigrationReactor.java +++ b/src/prerna/reactor/database/migration/SaveEngineMigrationReactor.java @@ -49,6 +49,7 @@ import prerna.engine.impl.rdbms.migration.MigrationStatus; import prerna.engine.impl.rdbms.migration.MigrationStatusUtils; import prerna.engine.impl.rdbms.migration.SchemaMigrationException; +import prerna.engine.impl.rdbms.migration.SchemaMigrationLock; import prerna.engine.impl.rdbms.migration.SchemaMigrationRunner; import prerna.reactor.AbstractReactor; import prerna.reactor.agent.mcp.MCPUtility; @@ -117,7 +118,19 @@ public NounMetadata execute() { } File migrationsFolder = MigrationFileUtils.getMigrationsFolder(rdbmsEngine); - String version = writeMigrationFile(migrationsFolder, sqlContent, description); + // Hold the same cross-node lock used by SchemaMigrationRunner while + // allocating the next version number and writing the file -- otherwise two + // concurrent saves can both scan the folder, compute the same "next + // version", and write two different files with a colliding version number + // (the second then fails at run time with a spurious checksum-mismatch + // error). Released before running the migration below so the runner's own + // lock acquisition isn't fighting a lock this same call already holds -- + // the lock-table insert path isn't reentrant, so holding across both steps + // would just make the runner start it as a retryable timeout every time. + String version; + try (SchemaMigrationLock lock = SchemaMigrationLock.acquire(rdbmsEngine)) { + version = writeMigrationFile(migrationsFolder, sqlContent, description); + } Map response = new HashMap<>(); response.put("version", version); @@ -128,7 +141,7 @@ public NounMetadata execute() { } catch (SchemaMigrationException e) { classLogger.error("Saved migration version '{}' for engine '{}' failed to run.", version, engineId, e); response.put("success", false); - response.put("errorMessage", latestErrorMessage(rdbmsEngine, migrationsFolder, version, e)); + response.put("errorMessage", handleRunFailure(rdbmsEngine, migrationsFolder, version, e)); } return new NounMetadata(response, PixelDataType.MAP); } @@ -175,12 +188,55 @@ private String sanitizeDescription(String description) { * (e.g. an out-of-order/checksum rejection never even reaches SQL * execution, so it never gets a history row -- in that case fall back to * the exception's own message). + *

+ * If OUR just-saved version is the one that actually failed, delete the + * file we just wrote instead of leaving a permanently-broken version at + * the head of the chain -- otherwise it (and the fail-closed design this + * feature already documents) blocks every later migration, and the engine + * itself, until someone finds and hand-edits that exact file. Nothing has + * recorded it as successfully applied, so it's always safe to discard; + * the attempt stays auditable regardless, since the runner's FAILED + * history row is untouched and simply shows as a {@code MISSING} status + * once its file is gone -- same "history row survives, file doesn't" + * shape as an old export missing its migrations folder. + *

+ * If a different, earlier pending version is what actually + * failed -- this one was never even reached -- keep the file: it's a + * legitimately valid, unattempted candidate, just queued behind a + * problem this save didn't cause. */ - private String latestErrorMessage(IRDBMSEngine engine, File migrationsFolder, String version, + // package-private (not private) so it's directly unit-testable without + // mocking the full execute() pipeline, same convention as + // SchemaMigrationLock.deriveLockKey / SchemaMigrationRunner.rejectIfPreviouslyFailedUnchanged + String handleRunFailure(IRDBMSEngine engine, File migrationsFolder, String version, SchemaMigrationException fallback) { List statuses = MigrationStatusUtils.getStatus(engine, migrationsFolder); - return statuses.stream().filter(s -> s.getVersion().equals(version)).findFirst() - .map(MigrationStatus::getErrorMessage).orElseGet(() -> fallback.getMessage()); + MigrationStatus ourStatus = statuses.stream().filter(s -> s.getVersion().equals(version)).findFirst() + .orElse(null); + if (ourStatus == null) { + return fallback.getMessage(); + } + if (ourStatus.getState() == MigrationStatus.State.FAILED) { + deleteMigrationFile(migrationsFolder, ourStatus.getFileName(), version); + return ourStatus.getErrorMessage() != null ? ourStatus.getErrorMessage() : fallback.getMessage(); + } + // still PENDING -- the loop never reached this version because an earlier, + // unrelated pending migration failed first; leave this file in place + return "This migration is valid but is queued behind an earlier pending migration that failed to " + + "apply. Resolve that one first (see the Migrations tab), then this version will run " + + "automatically on the next attempt: " + fallback.getMessage(); + } + + void deleteMigrationFile(File migrationsFolder, String fileName, String version) { + if (fileName == null) { + return; + } + try { + Files.deleteIfExists(migrationsFolder.toPath().resolve(fileName)); + } catch (IOException e) { + classLogger.error("Failed to delete failed migration file '{}' (version '{}') under '{}'.", fileName, + version, migrationsFolder.getAbsolutePath(), e); + } } @Override diff --git a/test/prerna/engine/impl/migration/SchemaMigrationRunnerUnitTests.java b/test/prerna/engine/impl/migration/SchemaMigrationRunnerUnitTests.java new file mode 100644 index 00000000000..e75ff929a1f --- /dev/null +++ b/test/prerna/engine/impl/migration/SchemaMigrationRunnerUnitTests.java @@ -0,0 +1,124 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.engine.impl.rdbms.migration; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.Timestamp; +import java.util.Collections; +import java.util.List; + +import org.junit.jupiter.api.Test; + +/** + * Covers {@link SchemaMigrationRunner#rejectIfPreviouslyFailedUnchanged} -- + * the short-circuit that stops a migration with an already-recorded failure + * from being re-executed against the live database on every subsequent + * {@code open()} (e.g. from unrelated read-only actions like browsing assets + * or viewing the smss), while still allowing a retry once the file content + * (and therefore its checksum) actually changes. + */ +public class SchemaMigrationRunnerUnitTests { + + private static final String VERSION = "3"; + private static final String FILE_NAME = "V3__broken_migration.sql"; + private static final String SQL_CONTENT = "ALTER TABLE ORDERS ADD COLUMN BAD_COLUMN NOT_A_REAL_TYPE;"; + + private MigrationFile migrationFile(String sqlContent) { + return new MigrationFile(VERSION, "broken_migration", FILE_NAME, sqlContent); + } + + private MigrationHistoryRecord failedRecord(String checksum, String errorMessage) { + return new MigrationHistoryRecord(VERSION, FILE_NAME, checksum, MigrationHistoryRecord.SYSTEM_APPLIED_BY, + new Timestamp(System.currentTimeMillis()), 12L, false, errorMessage); + } + + private MigrationHistoryRecord successRecord(String checksum) { + return new MigrationHistoryRecord(VERSION, FILE_NAME, checksum, MigrationHistoryRecord.SYSTEM_APPLIED_BY, + new Timestamp(System.currentTimeMillis()), 12L, true, null); + } + + @Test + void testThrowsWhenSameContentAlreadyFailed() { + MigrationFile migration = migrationFile(SQL_CONTENT); + String checksum = MigrationFileUtils.computeChecksum(SQL_CONTENT); + List history = List + .of(failedRecord(checksum, "ERROR: type \"not_a_real_type\" does not exist")); + + SchemaMigrationException ex = assertThrows(SchemaMigrationException.class, + () -> SchemaMigrationRunner.rejectIfPreviouslyFailedUnchanged(migration, history)); + assertTrue(ex.getMessage().contains(FILE_NAME)); + assertTrue(ex.getMessage().contains("not_a_real_type")); + } + + @Test + void testDoesNotThrowWhenContentChangedSinceLastFailure() { + MigrationFile migration = migrationFile(SQL_CONTENT); + // a different checksum -- as if the file's content was fixed since the + // recorded failure -- must NOT be treated as the same known failure + String staleChecksum = MigrationFileUtils.computeChecksum("some completely different sql content"); + List history = List.of(failedRecord(staleChecksum, "some earlier failure")); + + assertDoesNotThrow(() -> SchemaMigrationRunner.rejectIfPreviouslyFailedUnchanged(migration, history)); + } + + @Test + void testDoesNotThrowWhenNoHistoryForVersion() { + MigrationFile migration = migrationFile(SQL_CONTENT); + + assertDoesNotThrow( + () -> SchemaMigrationRunner.rejectIfPreviouslyFailedUnchanged(migration, Collections.emptyList())); + } + + @Test + void testDoesNotThrowWhenOnlyRecordForVersionSucceeded() { + MigrationFile migration = migrationFile(SQL_CONTENT); + String checksum = MigrationFileUtils.computeChecksum(SQL_CONTENT); + List history = List.of(successRecord(checksum)); + + // a successfully-applied version never reaches this check in + // runPendingMigrationsLocked (it's filtered into appliedVersions first), + // but this method itself should still be a no-op against a success row + assertDoesNotThrow(() -> SchemaMigrationRunner.rejectIfPreviouslyFailedUnchanged(migration, history)); + } + + @Test + void testDoesNotThrowWhenFailureRecordIsForADifferentVersion() { + MigrationFile migration = migrationFile(SQL_CONTENT); + String checksum = MigrationFileUtils.computeChecksum(SQL_CONTENT); + MigrationHistoryRecord otherVersionFailure = new MigrationHistoryRecord("2", "V2__add_customer_phone.sql", + checksum, MigrationHistoryRecord.SYSTEM_APPLIED_BY, new Timestamp(System.currentTimeMillis()), 12L, + false, "unrelated failure"); + + assertDoesNotThrow(() -> SchemaMigrationRunner.rejectIfPreviouslyFailedUnchanged(migration, + List.of(otherVersionFailure))); + } + +} diff --git a/test/prerna/reactor/database/migration/SaveEngineMigrationReactorUnitTests.java b/test/prerna/reactor/database/migration/SaveEngineMigrationReactorUnitTests.java new file mode 100644 index 00000000000..304c1fe3ba3 --- /dev/null +++ b/test/prerna/reactor/database/migration/SaveEngineMigrationReactorUnitTests.java @@ -0,0 +1,144 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.database.migration; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Timestamp; +import java.util.List; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.MockedStatic; + +import prerna.engine.api.IRDBMSEngine; +import prerna.engine.impl.rdbms.migration.MigrationStatus; +import prerna.engine.impl.rdbms.migration.MigrationStatusUtils; +import prerna.engine.impl.rdbms.migration.SchemaMigrationException; + +/** + * Covers {@link SaveEngineMigrationReactor#handleRunFailure} -- a failed + * migration should never leave a permanently-broken version at the head of + * the chain (blocking every later migration and the engine itself) when the + * failure is the newly-saved version's own fault, but a still-valid new file + * must be left alone if it was never even reached because an earlier, + * unrelated pending migration is what actually failed. + */ +public class SaveEngineMigrationReactorUnitTests { + + private static final String VERSION = "3"; + private static final String FILE_NAME = "V3__add_status_column.sql"; + + private SaveEngineMigrationReactor reactor; + private IRDBMSEngine engine; + + @TempDir + Path migrationsFolderPath; + + @BeforeEach + void setup() { + reactor = new SaveEngineMigrationReactor(); + engine = mock(IRDBMSEngine.class); + } + + @Test + void testDeletesFileWhenOurOwnVersionFailed() throws Exception { + File migrationsFolder = migrationsFolderPath.toFile(); + File savedFile = migrationsFolderPath.resolve(FILE_NAME).toFile(); + Files.writeString(savedFile.toPath(), "ALTER TABLE ORDERS ADD COLUMN BAD_COLUMN NOT_A_REAL_TYPE;"); + assertTrue(savedFile.exists()); + + MigrationStatus ourFailedStatus = new MigrationStatus(VERSION, "add_status_column", FILE_NAME, + MigrationStatus.State.FAILED, "SYSTEM", new Timestamp(System.currentTimeMillis()), 5L, + "ERROR: type \"not_a_real_type\" does not exist"); + SchemaMigrationException fallback = new SchemaMigrationException("Migration " + FILE_NAME + " failed: fallback"); + + try (MockedStatic statusMock = mockStatic(MigrationStatusUtils.class)) { + statusMock.when(() -> MigrationStatusUtils.getStatus(engine, migrationsFolder)) + .thenReturn(List.of(ourFailedStatus)); + + String errorMessage = reactor.handleRunFailure(engine, migrationsFolder, VERSION, fallback); + + assertEquals("ERROR: type \"not_a_real_type\" does not exist", errorMessage); + } + + assertFalse(savedFile.exists(), "the failed migration's own file should be deleted, not left blocking " + + "every later migration and the engine itself"); + } + + @Test + void testKeepsFileWhenAnEarlierUnrelatedVersionFailedInstead() throws Exception { + File migrationsFolder = migrationsFolderPath.toFile(); + File savedFile = migrationsFolderPath.resolve(FILE_NAME).toFile(); + Files.writeString(savedFile.toPath(), "ALTER TABLE ORDERS ADD COLUMN STATUS VARCHAR(50);"); + assertTrue(savedFile.exists()); + + // our version was never reached -- still PENDING, because an earlier + // version in the folder is the one that actually failed + MigrationStatus stillPending = new MigrationStatus(VERSION, "add_status_column", FILE_NAME, + MigrationStatus.State.PENDING, null, null, 0L, null); + SchemaMigrationException fallback = new SchemaMigrationException( + "Migration V2__broken.sql failed: some earlier problem"); + + try (MockedStatic statusMock = mockStatic(MigrationStatusUtils.class)) { + statusMock.when(() -> MigrationStatusUtils.getStatus(engine, migrationsFolder)) + .thenReturn(List.of(stillPending)); + + String errorMessage = reactor.handleRunFailure(engine, migrationsFolder, VERSION, fallback); + + assertTrue(errorMessage.contains("queued behind an earlier pending migration")); + assertTrue(errorMessage.contains("some earlier problem")); + } + + assertTrue(savedFile.exists(), "a valid, never-attempted migration must not be deleted just because an " + + "earlier, unrelated version failed first"); + } + + @Test + void testFallsBackToExceptionMessageWhenNoStatusFoundForVersion() { + File migrationsFolder = migrationsFolderPath.toFile(); + SchemaMigrationException fallback = new SchemaMigrationException("Unable to determine migration status"); + + try (MockedStatic statusMock = mockStatic(MigrationStatusUtils.class)) { + statusMock.when(() -> MigrationStatusUtils.getStatus(engine, migrationsFolder)).thenReturn(List.of()); + + String errorMessage = reactor.handleRunFailure(engine, migrationsFolder, VERSION, fallback); + + assertEquals("Unable to determine migration status", errorMessage); + } + } + +} From 54be969b0a40fc698591aaabc4db57029c1c4f51 Mon Sep 17 00:00:00 2001 From: Stella Bailey Date: Wed, 5 Aug 2026 08:52:15 -0400 Subject: [PATCH 3/3] feat: improve migration reliability --- .../migration/MigrationHistoryUtils.java | 75 ++++++++- .../rdbms/migration/MigrationStatusUtils.java | 23 ++- .../migration/SchemaMigrationException.java | 14 ++ .../migration/SchemaMigrationRunner.java | 91 +++------- .../DismissEngineMigrationRecordReactor.java | 159 ++++++++++++++++++ .../GetEngineMigrationFileReactor.java | 121 +++++++++++++ .../migration/SaveEngineMigrationReactor.java | 45 +++-- 7 files changed, 444 insertions(+), 84 deletions(-) create mode 100644 src/prerna/reactor/database/migration/DismissEngineMigrationRecordReactor.java create mode 100644 src/prerna/reactor/database/migration/GetEngineMigrationFileReactor.java diff --git a/src/prerna/engine/impl/rdbms/migration/MigrationHistoryUtils.java b/src/prerna/engine/impl/rdbms/migration/MigrationHistoryUtils.java index 869e388b130..b19fb1531d6 100644 --- a/src/prerna/engine/impl/rdbms/migration/MigrationHistoryUtils.java +++ b/src/prerna/engine/impl/rdbms/migration/MigrationHistoryUtils.java @@ -79,7 +79,7 @@ public static void ensureHistoryTable(IRDBMSEngine engine) { conn = engine.getConnection(); if (!queryUtil.tableExists(conn, HISTORY_TABLE, engine.getDatabase(), engine.getSchema())) { String createSql = queryUtil.createTable(HISTORY_TABLE, COL_NAMES, types); - classLogger.info("Creating migration history table for engine '{}' with sql {}", engine.getEngineId(), + classLogger.debug("Creating migration history table for engine '{}' with sql {}", engine.getEngineId(), createSql); engine.insertData(createSql); } @@ -161,11 +161,82 @@ public static void insertHistoryRow(Connection conn, MigrationHistoryRecord reco ps.setTimestamp(index++, record.getAppliedOn()); ps.setLong(index++, record.getExecutionTimeMs()); ps.setBoolean(index++, record.isSuccess()); - ps.setString(index++, record.getDescription()); + if (record.getDescription() != null) { + ps.setString(index++, record.getDescription()); + } else { + ps.setNull(index++, java.sql.Types.VARCHAR); + } ps.execute(); } } + /** + * Stores user-provided notes against the most recent successful history row for + * a given version. Called by {@code SaveEngineMigrationReactor} after a + * successful run, using the {@code notes} parameter the user supplied in the + * dialog. No-op if {@code notes} is null or blank. + * + * @param engine the engine whose history table should be updated + * @param version the version string to attach notes to (e.g. {@code "1"}) + * @param notes the user-provided description; skipped if null/blank + */ + public static void updateNotesForVersion(IRDBMSEngine engine, String version, String notes) { + if (notes == null || notes.isBlank()) { + return; + } + String updateSql = "UPDATE " + HISTORY_TABLE + " SET DESCRIPTION = ? WHERE VERSION = ? AND SUCCESS = ?"; + Connection conn = null; + try { + conn = engine.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(updateSql)) { + ps.setString(1, notes.trim()); + ps.setString(2, version); + ps.setBoolean(3, true); + ps.execute(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + } catch (SQLException e) { + classLogger.error("Failed to update notes for engine '{}', version '{}'.", + engine.getEngineId(), version, e); + throw new SchemaMigrationException("Unable to update notes for version " + version, e); + } finally { + ConnectionUtils.closeAllConnectionsIfPooling(engine, conn); + } + } + + /** + * Removes all {@code SEMOSS_SCHEMA_HISTORY} rows for a given version. Only + * safe to call for versions that are in a {@code MISSING} or + * {@code FAILED}-without-file state (the caller is responsible for verifying + * this before invoking). Deletes all rows for the version -- including older + * retry attempts -- so the version disappears entirely from the audit log. + * + * @param engine the engine whose history table should be modified + * @param version the version string to remove (e.g. {@code "1"}) + */ + public static void deleteHistoryForVersion(IRDBMSEngine engine, String version) { + String deleteSql = "DELETE FROM " + HISTORY_TABLE + " WHERE VERSION = ?"; + Connection conn = null; + try { + conn = engine.getConnection(); + try (PreparedStatement ps = conn.prepareStatement(deleteSql)) { + ps.setString(1, version); + ps.execute(); + } + if (!conn.getAutoCommit()) { + conn.commit(); + } + } catch (SQLException e) { + classLogger.error("Failed to delete migration history for engine '{}', version '{}'.", + engine.getEngineId(), version, e); + throw new SchemaMigrationException("Unable to delete migration history for version " + version, e); + } finally { + ConnectionUtils.closeAllConnectionsIfPooling(engine, conn); + } + } + /** * Records a run outcome on its own, independently-committed * connection/transaction -- used on the failure path, where the diff --git a/src/prerna/engine/impl/rdbms/migration/MigrationStatusUtils.java b/src/prerna/engine/impl/rdbms/migration/MigrationStatusUtils.java index b52f8ebd1b9..f1da0dccc3e 100644 --- a/src/prerna/engine/impl/rdbms/migration/MigrationStatusUtils.java +++ b/src/prerna/engine/impl/rdbms/migration/MigrationStatusUtils.java @@ -98,27 +98,42 @@ private static Map latestRecordPerVersion(List - * Editing the file changes its checksum, which naturally exits this - * short-circuit and treats it as a fresh attempt -- the same "fix the - * content to retry" mechanism {@link #verifyChecksumUnchanged} already - * relies on for the applied-version case, just applied to the failed - * case instead. - */ - // package-private (not private) so it's directly unit-testable without a - // real database, same convention as SchemaMigrationLock.deriveLockKey - static void rejectIfPreviouslyFailedUnchanged(MigrationFile migration, List history) { - String currentChecksum = MigrationFileUtils.computeChecksum(migration.getSqlContent()); - for (MigrationHistoryRecord record : history) { - if (!record.isSuccess() && record.getVersion().equals(migration.getVersion()) - && record.getChecksum().equals(currentChecksum)) { - throw new SchemaMigrationException("Migration " + migration.getFileName() - + " already failed with this exact content on " + record.getAppliedOn() + ": " - + record.getDescription() + ". Fix the SQL -- which changes its checksum -- to retry, " - + "or check SEMOSS_SCHEMA_HISTORY for full details."); + + "Restore the original file content or create a new version instead of editing it.", + migration.getVersion()); } } } @@ -242,15 +218,14 @@ private static void rejectIfOutOfOrder(MigrationFile migration, String highestAp && MigrationFileUtils.compareVersions(migration.getVersion(), highestAppliedVersion) < 0) { throw new SchemaMigrationException("Migration " + migration.getFileName() + " has version " + migration.getVersion() + ", which is lower than the highest already-applied version " - + highestAppliedVersion + ". Out-of-order migrations are not supported."); + + highestAppliedVersion + ". Out-of-order migrations are not supported.", + migration.getVersion()); } } - private static void runMigration(IRDBMSEngine engine, MigrationFile migration) { + private static void runMigration(IRDBMSEngine engine, MigrationFile migration, String appliedBy) { long start = System.currentTimeMillis(); String checksum = MigrationFileUtils.computeChecksum(migration.getSqlContent()); - boolean success = false; - String failureReason = null; Connection conn = null; try { // Set/commit autocommit on the specific borrowed connection, never via @@ -270,41 +245,29 @@ private static void runMigration(IRDBMSEngine engine, MigrationFile migration) { } long executionTimeMs = System.currentTimeMillis() - start; MigrationHistoryRecord record = new MigrationHistoryRecord(migration.getVersion(), migration.getFileName(), - checksum, MigrationHistoryRecord.SYSTEM_APPLIED_BY, new Timestamp(System.currentTimeMillis()), + checksum, appliedBy, new Timestamp(System.currentTimeMillis()), executionTimeMs, true, null); // Insert on the SAME connection/transaction as the migration's own SQL so // both commit -- or both roll back -- together. Confirmed from Flyway's own // source (SchemaHistory.java: "a migration failure automatically triggers a // rollback of all changes, including the ones in the schema history table") // that this pairing is the real, correct behavior -- recording success on a - // separate connection/transaction (the earlier version of this method) left a - // durability gap: a crash between the two commits would leave the migration - // permanently applied with no history row, causing it to look "pending" and - // be retried against non-idempotent DDL next time. + // separate connection/transaction left a durability gap: a crash between the + // two commits would leave the migration permanently applied with no history + // row, causing it to look "pending" and be retried against non-idempotent DDL. MigrationHistoryUtils.insertHistoryRow(conn, record); conn.commit(); - success = true; } catch (Exception e) { classLogger.error("Migration '{}' failed against engine '{}'.", migration.getFileName(), engine.getEngineId(), e); - failureReason = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); rollbackQuietly(conn, migration); + String failureReason = e.getMessage() != null ? e.getMessage() : e.getClass().getSimpleName(); + throw new SchemaMigrationException( + "Migration " + migration.getFileName() + " failed: " + failureReason, + migration.getVersion()); } finally { ConnectionUtils.closeAllConnectionsIfPooling(engine, conn); } - - if (!success) { - // the migration's own transaction (and any history row attempted inside it) - // was just rolled back -- record the failure on its own fresh connection so - // it's actually durable and visible in the Migrations tab - long executionTimeMs = System.currentTimeMillis() - start; - MigrationHistoryRecord failureRecord = new MigrationHistoryRecord(migration.getVersion(), - migration.getFileName(), checksum, MigrationHistoryRecord.SYSTEM_APPLIED_BY, - new Timestamp(System.currentTimeMillis()), executionTimeMs, false, failureReason); - MigrationHistoryUtils.recordMigration(engine, failureRecord); - throw new SchemaMigrationException( - "Migration " + migration.getFileName() + " failed: " + failureReason); - } } private static void rollbackQuietly(Connection conn, MigrationFile migration) { diff --git a/src/prerna/reactor/database/migration/DismissEngineMigrationRecordReactor.java b/src/prerna/reactor/database/migration/DismissEngineMigrationRecordReactor.java new file mode 100644 index 00000000000..93a0fd843ee --- /dev/null +++ b/src/prerna/reactor/database/migration/DismissEngineMigrationRecordReactor.java @@ -0,0 +1,159 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.database.migration; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityQueryUtils; +import prerna.engine.api.IDatabaseEngine; +import prerna.engine.api.IRDBMSEngine; +import prerna.engine.impl.rdbms.migration.MigrationFileUtils; +import prerna.engine.impl.rdbms.migration.MigrationHistoryUtils; +import prerna.engine.impl.rdbms.migration.MigrationStatus; +import prerna.engine.impl.rdbms.migration.MigrationStatusUtils; +import prerna.reactor.AbstractReactor; +import prerna.reactor.agent.mcp.MCPUtility; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Removes all {@code SEMOSS_SCHEMA_HISTORY} rows for a given version, clearing + * it from the Migrations tab. Allowed for {@code MISSING} and {@code FAILED} + * versions -- both represent cases where no schema change was permanently + * applied. For {@code FAILED} versions whose SQL file still exists on disk, the + * file is also deleted so the version can be rewritten from scratch. + * Dismissing a {@code SUCCESS} or {@code OUTDATED} version is rejected because + * the history row is the only durable record that those DDL changes ever ran. + * + * 

+ * DismissEngineMigrationRecord(engine = ["<engineId>"], version = ["1"]);
+ *
+ * + * Returns: {@code true} on success. + */ +public class DismissEngineMigrationRecordReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(DismissEngineMigrationRecordReactor.class); + + public DismissEngineMigrationRecordReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.ENGINE.getKey(), ReactorKeysEnum.VERSION.getKey() }; + this.keyRequired = new int[] { 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String rawEngineId = this.keyValue.get(this.keysToGet[0]); + String version = this.keyValue.get(this.keysToGet[1]); + if (rawEngineId == null || rawEngineId.trim().isEmpty()) { + throw new IllegalArgumentException("Must provide an engine id"); + } + if (version == null || version.trim().isEmpty()) { + throw new IllegalArgumentException("Must provide a version to dismiss"); + } + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("User must be logged in"); + } + String engineId = SecurityQueryUtils.testUserEngineIdForAlias(user, rawEngineId); + if (!SecurityEngineUtils.userCanEditEngine(user, engineId)) { + throw new IllegalArgumentException( + "Engine " + engineId + " does not exist or user does not have access to edit it"); + } + + IDatabaseEngine database = Utility.getDatabase(engineId); + if (database == null) { + throw new IllegalArgumentException("Engine " + engineId + " could not be loaded"); + } + if (!(database instanceof IRDBMSEngine rdbmsEngine)) { + throw new IllegalArgumentException("Engine " + engineId + " is not a JDBC database engine"); + } + + String trimmedVersion = version.trim(); + File migrationsFolder = MigrationFileUtils.getMigrationsFolder(rdbmsEngine); + List statuses = MigrationStatusUtils.getStatus(rdbmsEngine, migrationsFolder); + MigrationStatus target = statuses.stream() + .filter(s -> s.getVersion().equals(trimmedVersion)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("No migration record found for version " + trimmedVersion)); + + MigrationStatus.State state = target.getState(); + boolean isDismissable = state == MigrationStatus.State.MISSING + || state == MigrationStatus.State.FAILED; + if (!isDismissable) { + throw new IllegalArgumentException( + "Migration version " + trimmedVersion + " is in state " + state + + " and cannot be dismissed. Only MISSING and FAILED versions are eligible for dismissal."); + } + + // For FAILED rows that still have a file on disk, delete the file so the + // version is completely reset and does not reappear as PENDING after dismiss. + if (state == MigrationStatus.State.FAILED && target.getFileName() != null) { + try { + Files.deleteIfExists(migrationsFolder.toPath().resolve(target.getFileName())); + } catch (IOException e) { + classLogger.warn("Could not delete migration file '{}' for version '{}' during dismiss.", + target.getFileName(), trimmedVersion, e); + } + } + + classLogger.info("User '{}' dismissing migration history for engine '{}', version '{}'.", + user.getPrimaryLoginToken() != null ? user.getPrimaryLoginToken().getId() : "unknown", + engineId, trimmedVersion); + MigrationHistoryUtils.deleteHistoryForVersion(rdbmsEngine, trimmedVersion); + return new NounMetadata(true, PixelDataType.BOOLEAN); + } + + @Override + public String getReactorDescription() { + return "Removes a MISSING or FAILED migration history record from an engine's schema history table"; + } + + @Override + public Map getMcpToolMetadata() { + Map meta = new HashMap<>(); + // permanently deletes audit history -- must never be agent-auto-triggerable + meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + meta.put(MCPUtility.UI_DISPLAY_LOCATION, MCPUtility.MCPDisplayOption.SIDEBAR.getValue()); + return meta; + } + +} diff --git a/src/prerna/reactor/database/migration/GetEngineMigrationFileReactor.java b/src/prerna/reactor/database/migration/GetEngineMigrationFileReactor.java new file mode 100644 index 00000000000..8c15026327a --- /dev/null +++ b/src/prerna/reactor/database/migration/GetEngineMigrationFileReactor.java @@ -0,0 +1,121 @@ +/******************************************************************************* + * Copyright 2015 Defense Health Agency (DHA) + * + * If your use of this software does not include any GPLv2 components: + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---------------------------------------------------------------------------- + * If your use of this software includes any GPLv2 components: + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + *******************************************************************************/ +package prerna.reactor.database.migration; + +import java.io.File; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import prerna.auth.User; +import prerna.auth.utils.SecurityEngineUtils; +import prerna.auth.utils.SecurityQueryUtils; +import prerna.engine.api.IDatabaseEngine; +import prerna.engine.api.IRDBMSEngine; +import prerna.engine.impl.rdbms.migration.MigrationFile; +import prerna.engine.impl.rdbms.migration.MigrationFileUtils; +import prerna.reactor.AbstractReactor; +import prerna.sablecc2.om.PixelDataType; +import prerna.sablecc2.om.ReactorKeysEnum; +import prerna.sablecc2.om.nounmeta.NounMetadata; +import prerna.util.Utility; + +/** + * Returns the raw SQL content of a specific migration file -- used by the + * Migrations tab to let users view a file's SQL without editing it. READ-only; + * does not run, modify, or record anything. + * + * 
+ * GetEngineMigrationFile(engine = ["<engineId>"], version = ["1"]);
+ *
+ * + * Returns: the SQL content as a plain string, or throws if the file does not + * exist on disk (e.g. FAILED or MISSING state rows whose file was removed). + */ +public class GetEngineMigrationFileReactor extends AbstractReactor { + + private static final Logger classLogger = LogManager.getLogger(GetEngineMigrationFileReactor.class); + + public GetEngineMigrationFileReactor() { + this.keysToGet = new String[] { ReactorKeysEnum.ENGINE.getKey(), ReactorKeysEnum.VERSION.getKey() }; + this.keyRequired = new int[] { 1, 1 }; + } + + @Override + public NounMetadata execute() { + organizeKeys(); + String rawEngineId = this.keyValue.get(this.keysToGet[0]); + String version = this.keyValue.get(this.keysToGet[1]); + if (rawEngineId == null || rawEngineId.trim().isEmpty()) { + throw new IllegalArgumentException("Must provide an engine id"); + } + if (version == null || version.trim().isEmpty()) { + throw new IllegalArgumentException("Must provide a version"); + } + + User user = this.insight.getUser(); + if (user == null) { + throw new IllegalArgumentException("User must be logged in"); + } + String engineId = SecurityQueryUtils.testUserEngineIdForAlias(user, rawEngineId); + if (!SecurityEngineUtils.userCanViewEngine(user, engineId)) { + throw new IllegalArgumentException( + "Engine " + engineId + " does not exist or user does not have access to view it"); + } + + IDatabaseEngine database = Utility.getDatabase(engineId); + if (database == null) { + throw new IllegalArgumentException("Engine " + engineId + " could not be loaded"); + } + if (!(database instanceof IRDBMSEngine rdbmsEngine)) { + throw new IllegalArgumentException("Engine " + engineId + " is not a JDBC database engine"); + } + + String trimmedVersion = version.trim(); + File migrationsFolder = MigrationFileUtils.getMigrationsFolder(rdbmsEngine); + List files = MigrationFileUtils.scanMigrationsFolder(migrationsFolder); + MigrationFile file = files.stream() + .filter(f -> f.getVersion().equals(trimmedVersion)) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException( + "No migration file found for version " + trimmedVersion + + ". The file may have been removed (check the migration state in the Migrations tab).")); + + classLogger.debug("User '{}' viewing migration file for engine '{}', version '{}'.", + user.getPrimaryLoginToken() != null ? user.getPrimaryLoginToken().getId() : "unknown", + engineId, trimmedVersion); + return new NounMetadata(file.getSqlContent(), PixelDataType.CONST_STRING); + } + + @Override + public String getReactorDescription() { + return "Returns the raw SQL content of a migration file for a given engine and version number"; + } + +} diff --git a/src/prerna/reactor/database/migration/SaveEngineMigrationReactor.java b/src/prerna/reactor/database/migration/SaveEngineMigrationReactor.java index 423dbf99234..682231e1d95 100644 --- a/src/prerna/reactor/database/migration/SaveEngineMigrationReactor.java +++ b/src/prerna/reactor/database/migration/SaveEngineMigrationReactor.java @@ -46,6 +46,8 @@ import prerna.engine.api.IRDBMSEngine; import prerna.engine.impl.rdbms.migration.MigrationFile; import prerna.engine.impl.rdbms.migration.MigrationFileUtils; +import prerna.engine.impl.rdbms.migration.MigrationHistoryRecord; +import prerna.engine.impl.rdbms.migration.MigrationHistoryUtils; import prerna.engine.impl.rdbms.migration.MigrationStatus; import prerna.engine.impl.rdbms.migration.MigrationStatusUtils; import prerna.engine.impl.rdbms.migration.SchemaMigrationException; @@ -80,10 +82,12 @@ public class SaveEngineMigrationReactor extends AbstractReactor { private static final Logger classLogger = LogManager.getLogger(SaveEngineMigrationReactor.class); + private static final String NOTES_KEY = "notes"; + public SaveEngineMigrationReactor() { this.keysToGet = new String[] { ReactorKeysEnum.ENGINE.getKey(), ReactorKeysEnum.SQL.getKey(), - ReactorKeysEnum.DESCRIPTION.getKey() }; - this.keyRequired = new int[] { 1, 1, 1 }; + ReactorKeysEnum.DESCRIPTION.getKey(), NOTES_KEY }; + this.keyRequired = new int[] { 1, 1, 1, 0 }; } @Override @@ -92,6 +96,7 @@ public NounMetadata execute() { String rawEngineId = this.keyValue.get(this.keysToGet[0]); String sqlContent = this.keyValue.get(this.keysToGet[1]); String description = this.keyValue.get(this.keysToGet[2]); + String notes = this.keyValue.get(NOTES_KEY); if (rawEngineId == null || rawEngineId.trim().isEmpty()) { throw new IllegalArgumentException("Must provide an engine id to save a migration against"); } @@ -113,6 +118,9 @@ public NounMetadata execute() { } IDatabaseEngine database = Utility.getDatabase(engineId); + if (database == null) { + throw new IllegalArgumentException("Engine " + engineId + " could not be loaded"); + } if (!(database instanceof IRDBMSEngine rdbmsEngine)) { throw new IllegalArgumentException("Engine " + engineId + " is not a JDBC database engine"); } @@ -132,10 +140,15 @@ public NounMetadata execute() { version = writeMigrationFile(migrationsFolder, sqlContent, description); } + String appliedBy = (user.getPrimaryLoginToken() != null) + ? user.getPrimaryLoginToken().getId() + : MigrationHistoryRecord.SYSTEM_APPLIED_BY; + Map response = new HashMap<>(); response.put("version", version); try { - SchemaMigrationRunner.runPendingMigrations(rdbmsEngine, migrationsFolder); + SchemaMigrationRunner.runPendingMigrations(rdbmsEngine, migrationsFolder, appliedBy); + MigrationHistoryUtils.updateNotesForVersion(rdbmsEngine, version, notes); response.put("success", true); response.put("errorMessage", null); } catch (SchemaMigrationException e) { @@ -207,21 +220,24 @@ private String sanitizeDescription(String description) { */ // package-private (not private) so it's directly unit-testable without // mocking the full execute() pipeline, same convention as - // SchemaMigrationLock.deriveLockKey / SchemaMigrationRunner.rejectIfPreviouslyFailedUnchanged + // SchemaMigrationLock.deriveLockKey String handleRunFailure(IRDBMSEngine engine, File migrationsFolder, String version, SchemaMigrationException fallback) { - List statuses = MigrationStatusUtils.getStatus(engine, migrationsFolder); - MigrationStatus ourStatus = statuses.stream().filter(s -> s.getVersion().equals(version)).findFirst() - .orElse(null); - if (ourStatus == null) { + String failedVersion = fallback.getFailedVersion(); + if (failedVersion == null || version.equals(failedVersion)) { + // our version was the direct failure (or infrastructure failed before any + // version check) -- delete the file so this version does not re-appear as + // PENDING on the next engine open + List statuses = MigrationStatusUtils.getStatus(engine, migrationsFolder); + MigrationStatus ourStatus = statuses.stream().filter(s -> s.getVersion().equals(version)).findFirst() + .orElse(null); + if (ourStatus != null) { + deleteMigrationFile(migrationsFolder, ourStatus.getFileName(), version); + } return fallback.getMessage(); } - if (ourStatus.getState() == MigrationStatus.State.FAILED) { - deleteMigrationFile(migrationsFolder, ourStatus.getFileName(), version); - return ourStatus.getErrorMessage() != null ? ourStatus.getErrorMessage() : fallback.getMessage(); - } - // still PENDING -- the loop never reached this version because an earlier, - // unrelated pending migration failed first; leave this file in place + // an earlier pending migration failed before ours was reached -- keep our file + // since it has not been attempted and will run once the blocker is resolved return "This migration is valid but is queued behind an earlier pending migration that failed to " + "apply. Resolve that one first (see the Migrations tab), then this version will run " + "automatically on the next attempt: " + fallback.getMessage(); @@ -245,6 +261,7 @@ public Map getMcpToolMetadata() { // this creates AND immediately runs arbitrary DDL/DML -- must never be // agent-auto-triggerable meta.put(MCPUtility.SMSS_MCP_EXECUTION, MCPUtility.MCPExecution.ASK.getValue()); + meta.put(MCPUtility.UI_DISPLAY_LOCATION, MCPUtility.MCPDisplayOption.SIDEBAR.getValue()); return meta; }