From b872ba8df0b3bbda20a8914373b011e873fe49d6 Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Tue, 8 Jul 2025 21:20:55 +0500 Subject: [PATCH 01/10] major spock version upgrade command (draft) --- cli/scripts/um.py | 156 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 154 insertions(+), 2 deletions(-) diff --git a/cli/scripts/um.py b/cli/scripts/um.py index f07aedbc..9c4a35db 100755 --- a/cli/scripts/um.py +++ b/cli/scripts/um.py @@ -3,7 +3,8 @@ import os, sys, glob, sqlite3, time import fire, meta, util - +from cluster import load_json, get_cluster_json # helper functions for reading cluster JSON +import subprocess isJSON = util.isJSON MY_HOME = util.MY_HOME @@ -235,6 +236,156 @@ def verify_metadata(Project="", Stage="prod", IsCurrent=0): meta.pretty_sql(sql) + +import os +import subprocess + +def upgrade_spock(cluster_name: str, new_spock_ver: str): + """ + Upgrade the Spock extension cluster-wide. + + ./pgedge um upgrade-spock + """ + # 1. Warn that downgrades (or same‐version/minor upgrades) aren’t supported + util.message( + "WARNING: Downgrading to a previous Spock version—or upgrading within the same major version—is not supported; " + "only upgrades to a newer major version are allowed.", + "warn", + isJSON, + ) + + # 2. Load cluster JSON + util.message(f"## Loading cluster '{cluster_name}' JSON definition file", "info", isJSON) + try: + db, db_settings, nodes = load_json(cluster_name) + except Exception as e: + util.exit_message(f"Unable to load cluster JSON: {e}", 1, isJSON) + + if not nodes: + parsed = get_cluster_json(cluster_name) + if not parsed or not parsed.get("nodes"): + util.exit_message("Unable to load cluster JSON", 1, isJSON) + nodes = parsed["nodes"] + + # 3. Extract node directories + node_dirs = [] + for n in nodes: + dir_ = n.get("node_dir") or n.get("dir") or n.get("path") + if dir_: + node_dirs.append(dir_) + if not node_dirs: + util.exit_message("No node_dir entries found in cluster JSON.", 1, isJSON) + util.message(f"Discovered node directories: {', '.join(node_dirs)}", "info", isJSON) + + # 4. Auto-detect current Spock version from JSON + current_spock_ver = db_settings.get("spock_version") + if not current_spock_ver: + util.exit_message( + "Current Spock version not found in cluster JSON (expected db_settings['spock_version']).", + 1, + isJSON + ) + util.message( + f"Detected current Spock version from JSON: {current_spock_ver}", + "info", + isJSON + ) + + # 5. Refuse no-ops + if new_spock_ver == current_spock_ver: + util.exit_message( + f"New Spock version {new_spock_ver} is the same as the current version.", + 1, + isJSON + ) + + # 5.1 Compare major versions as strings + try: + current_major = current_spock_ver.split(".")[0] + new_major = new_spock_ver.split(".")[0] + except IndexError: + util.exit_message( + f"Invalid Spock version format. Got current='{current_spock_ver}', requested='{new_spock_ver}'.", + 1, + isJSON + ) + + if new_major <= current_major: + util.exit_message( + f"Invalid upgrade: new Spock major version {new_major} must be greater than current major version {current_major}.", + 1, + isJSON + ) + + # 6. If upgrading to Spock 5, install Spock5 on each node + if new_major == "5": + for dir_ in node_dirs: + pgedge_dir = os.path.join(dir_, "pgedge") + util.message( + f"Installing Spock5 on node at '{pgedge_dir}'", + "info", + isJSON + ) + try: + subprocess.run(["./pgedge", "um", "install", "spock5"], cwd=pgedge_dir, check=True) + except subprocess.CalledProcessError as e: + util.exit_message( + f"Spock5 installation failed in '{pgedge_dir}': {e}", + 1, + isJSON + ) + + # 7. Warn about downtime + util.message( + f"WARNING: Upgrading Spock on cluster '{cluster_name}' " + f"from {current_spock_ver} to {new_spock_ver} will cause downtime; " + "all nodes will be restarted.", + "warn", + isJSON, + ) + + # 8. Check for Backrest configuration and run pgBackRest backups + for n in nodes: + br = n.get("backrest") + if br: + stanza = br.get("stanza") + node_name = n.get("name") or n.get("node_dir") or "unknown" + node_dir = n.get("node_dir") or n.get("dir") or n.get("path") + pgedge_dir = os.path.join(node_dir, "pgedge") + + util.message( + f"Running pgBackRest backup for stanza '{stanza}' on node '{node_name}'", + "info", + isJSON + ) + try: + subprocess.run(["./pgedge", "backrest", "backup", stanza], cwd=pgedge_dir, check=True) + except subprocess.CalledProcessError as e: + util.exit_message( + f"pgBackRest backup failed on node '{node_name}': {e}", + 1, + isJSON + ) + + # 9. Determine the running PostgreSQL version + try: + pg_version = db_settings.get("pg_version") or util.fetch_pg_version(node_dirs[0]) + util.message(f"Detected PostgreSQL version: {pg_version}", "info", isJSON) + except Exception as e: + util.exit_message(f"Unable to determine PostgreSQL version: {e}", 1, isJSON) + + # 10. Validate compatibility + util.validate_spock_pg_compat(current_spock_ver, pg_version) + + # 11. Success + util.exit_message( + f"Successfully upgraded Spock to {new_spock_ver} on cluster '{cluster_name}'.", + 0, + isJSON + ) + + + if __name__ == "__main__": fire.Fire( { @@ -245,6 +396,7 @@ def verify_metadata(Project="", Stage="prod", IsCurrent=0): "upgrade": upgrade, "clean": clean, "verify-metadata": verify_metadata, - "download": download, + "download": download, + "upgrade-spock": upgrade_spock, } ) From efcbfbaa36afe6a4f6442fb95e326cef87e516be Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Wed, 9 Jul 2025 17:31:10 +0500 Subject: [PATCH 02/10] successfully upgraded to spock5 using um upgrade-spock --- cli/scripts/um.py | 169 ++++++++++++++++++++++++---------------------- 1 file changed, 89 insertions(+), 80 deletions(-) diff --git a/cli/scripts/um.py b/cli/scripts/um.py index 9c4a35db..d513f167 100755 --- a/cli/scripts/um.py +++ b/cli/scripts/um.py @@ -237,16 +237,13 @@ def verify_metadata(Project="", Stage="prod", IsCurrent=0): -import os -import subprocess - def upgrade_spock(cluster_name: str, new_spock_ver: str): """ Upgrade the Spock extension cluster-wide. ./pgedge um upgrade-spock """ - # 1. Warn that downgrades (or same‐version/minor upgrades) aren’t supported + # 1. Warn that downgrades/same‐version upgrades aren’t supported util.message( "WARNING: Downgrading to a previous Spock version—or upgrading within the same major version—is not supported; " "only upgrades to a newer major version are allowed.", @@ -254,130 +251,142 @@ def upgrade_spock(cluster_name: str, new_spock_ver: str): isJSON, ) - # 2. Load cluster JSON - util.message(f"## Loading cluster '{cluster_name}' JSON definition file", "info", isJSON) + # 2. Load cluster JSON (db list, settings, and node definitions) + util.message(f"## Loading cluster '{cluster_name}' JSON definition", "info", isJSON) try: - db, db_settings, nodes = load_json(cluster_name) + db_list, db_settings, nodes = load_json(cluster_name) except Exception as e: util.exit_message(f"Unable to load cluster JSON: {e}", 1, isJSON) - if not nodes: - parsed = get_cluster_json(cluster_name) - if not parsed or not parsed.get("nodes"): - util.exit_message("Unable to load cluster JSON", 1, isJSON) - nodes = parsed["nodes"] + util.exit_message("No 'nodes' found in cluster JSON.", 1, isJSON) + if not db_list: + util.exit_message("No 'databases' defined in cluster JSON.", 1, isJSON) - # 3. Extract node directories - node_dirs = [] - for n in nodes: - dir_ = n.get("node_dir") or n.get("dir") or n.get("path") - if dir_: - node_dirs.append(dir_) - if not node_dirs: - util.exit_message("No node_dir entries found in cluster JSON.", 1, isJSON) - util.message(f"Discovered node directories: {', '.join(node_dirs)}", "info", isJSON) - - # 4. Auto-detect current Spock version from JSON + # 3. Extract current Spock version current_spock_ver = db_settings.get("spock_version") if not current_spock_ver: util.exit_message( - "Current Spock version not found in cluster JSON (expected db_settings['spock_version']).", + "Current Spock version not found in cluster JSON.", 1, isJSON ) - util.message( - f"Detected current Spock version from JSON: {current_spock_ver}", - "info", - isJSON - ) + util.message(f"Detected current Spock version from JSON: {current_spock_ver}", "info", isJSON) - # 5. Refuse no-ops + # 4. Validate requested upgrade if new_spock_ver == current_spock_ver: util.exit_message( f"New Spock version {new_spock_ver} is the same as the current version.", 1, isJSON ) - - # 5.1 Compare major versions as strings try: - current_major = current_spock_ver.split(".")[0] - new_major = new_spock_ver.split(".")[0] - except IndexError: + current_major = int(current_spock_ver.split('.')[0]) + new_major = int(new_spock_ver.split('.')[0]) + except (IndexError, ValueError): util.exit_message( - f"Invalid Spock version format. Got current='{current_spock_ver}', requested='{new_spock_ver}'.", + f"Invalid version format: current='{current_spock_ver}', requested='{new_spock_ver}'.", 1, isJSON ) - if new_major <= current_major: util.exit_message( - f"Invalid upgrade: new Spock major version {new_major} must be greater than current major version {current_major}.", + f"Invalid upgrade: new Spock major version {new_major} must be greater than current {current_major}.", 1, isJSON ) - # 6. If upgrading to Spock 5, install Spock5 on each node - if new_major == "5": - for dir_ in node_dirs: - pgedge_dir = os.path.join(dir_, "pgedge") + # 5. Gather node directories + node_dirs = [] + for n in nodes: + dir_ = n.get('path') + if dir_: + node_dirs.append(dir_) + if not node_dirs: + util.exit_message("Node directories not defined in JSON.", 1, isJSON) + util.message(f"Discovered node directories: {', '.join(node_dirs)}", "info", isJSON) + + # 6. Install new Spock major if needed + if new_major > current_major: + for base in node_dirs: + pgedge_dir = os.path.join(base, 'pgedge') util.message( - f"Installing Spock5 on node at '{pgedge_dir}'", - "info", - isJSON + f"Installing Spock{new_major} on node '{pgedge_dir}'", "info", isJSON ) try: - subprocess.run(["./pgedge", "um", "install", "spock5"], cwd=pgedge_dir, check=True) + subprocess.run(['./pgedge', 'um', 'install', f'spock{new_major}'], cwd=pgedge_dir, check=True) except subprocess.CalledProcessError as e: - util.exit_message( - f"Spock5 installation failed in '{pgedge_dir}': {e}", - 1, - isJSON - ) + util.exit_message(f"Spock install failed: {e}", 1, isJSON) - # 7. Warn about downtime + # 7. Downtime warning util.message( - f"WARNING: Upgrading Spock on cluster '{cluster_name}' " - f"from {current_spock_ver} to {new_spock_ver} will cause downtime; " - "all nodes will be restarted.", + f"WARNING: Upgrading Spock on '{cluster_name}' from {current_spock_ver} to {new_spock_ver} will cause downtime.", "warn", - isJSON, + isJSON ) - # 8. Check for Backrest configuration and run pgBackRest backups + # 8. Backup with pgBackRest if configured for n in nodes: - br = n.get("backrest") + br = n.get('backrest') if br: - stanza = br.get("stanza") - node_name = n.get("name") or n.get("node_dir") or "unknown" - node_dir = n.get("node_dir") or n.get("dir") or n.get("path") - pgedge_dir = os.path.join(node_dir, "pgedge") - - util.message( - f"Running pgBackRest backup for stanza '{stanza}' on node '{node_name}'", - "info", - isJSON - ) + stanza = br.get('stanza') + node_dir = n['path'] + pgedge_dir = os.path.join(node_dir, 'pgedge') + util.message(f"Running backup stanza '{stanza}'", "info", isJSON) try: - subprocess.run(["./pgedge", "backrest", "backup", stanza], cwd=pgedge_dir, check=True) + subprocess.run(['./pgedge', 'backrest', 'backup', stanza], cwd=pgedge_dir, check=True) except subprocess.CalledProcessError as e: - util.exit_message( - f"pgBackRest backup failed on node '{node_name}': {e}", - 1, - isJSON - ) + util.exit_message(f"Backup failed on node {n.get('name')}: {e}", 1, isJSON) - # 9. Determine the running PostgreSQL version + # 9. Detect PostgreSQL version try: - pg_version = db_settings.get("pg_version") or util.fetch_pg_version(node_dirs[0]) + pg_version = db_settings.get('pg_version') or util.fetch_pg_version(node_dirs[0]) util.message(f"Detected PostgreSQL version: {pg_version}", "info", isJSON) except Exception as e: - util.exit_message(f"Unable to determine PostgreSQL version: {e}", 1, isJSON) + util.exit_message(f"Could not detect PostgreSQL version: {e}", 1, isJSON) + + # 10. Compatibility check + util.validate_spock_pg_compat(new_spock_ver, pg_version) + + # 11. Perform ALTER EXTENSION for each db on each node + for n in nodes: + node_name = n.get('name') or n['path'] + node_dir = n['path'] + port = n.get('port') or db_settings.get('port') + if not port: + util.exit_message(f"Port missing for node '{node_name}'", 1, isJSON) + + # Build path to psql + pg_bin = f"pg{pg_version}" + psql_exec = os.path.join(node_dir, 'pgedge', pg_bin, 'bin', 'psql') + - # 10. Validate compatibility - util.validate_spock_pg_compat(current_spock_ver, pg_version) + for db in db_list: + db_name = db.get('db_name') + db_user = db.get('db_user') + if not db_name or not db_user: + util.exit_message( + f"Database entry incomplete in JSON for node '{node_name}'", 1, isJSON + ) + util.message( + f"Altering Spock in {db_name}@{node_name} as {db_user}", + "info", + isJSON + ) + try: + subprocess.run( + [ + psql_exec, + '-p', str(port), + '-U', db_user, + '-d', db_name, + '-c', f"ALTER EXTENSION spock UPDATE TO \"{new_spock_ver}\";" + ], + check=True + ) + except subprocess.CalledProcessError as e: + util.exit_message(f"ALTER EXTENSION failed: {e}", 1, isJSON) - # 11. Success + # 12. Complete util.exit_message( f"Successfully upgraded Spock to {new_spock_ver} on cluster '{cluster_name}'.", 0, From a723b05c27860ab375e6713ebe19593393f33474 Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Mon, 21 Jul 2025 16:56:42 +0500 Subject: [PATCH 03/10] spock 5 upgrade support --- cli/scripts/um.py | 215 +++++++++++++--------------------------------- 1 file changed, 62 insertions(+), 153 deletions(-) diff --git a/cli/scripts/um.py b/cli/scripts/um.py index d513f167..52b12ff9 100755 --- a/cli/scripts/um.py +++ b/cli/scripts/um.py @@ -6,6 +6,8 @@ from cluster import load_json, get_cluster_json # helper functions for reading cluster JSON import subprocess isJSON = util.isJSON +import re +import sqlite3 as _sqlite3 MY_HOME = util.MY_HOME @@ -84,16 +86,23 @@ def update(): run_cmd("update") + def install(component, active=True): - """Install a component.""" + """Install a component or trigger Spock upgrade for spock50-pg.""" + # Only invoke upgrade_spock when installing spock50-pg* + if component.startswith("spock50-pg"): + print(f"Detected Spock upgrade target '{component}', invoking upgrade_spock()...\n") + upgrade_spock() + # After successful compatibility, proceed with install + util.message(f"um.install(install {component})", "debug") + run_cmd("install", component) + return if active not in (True, False): util.exit_message("'active' parm must be True or False") - cmd = "install" - if active is False: + if not active: cmd = "install --no-preload" - util.message(f"um.install({cmd} {component})", "debug") run_cmd(cmd, component) @@ -237,163 +246,63 @@ def verify_metadata(Project="", Stage="prod", IsCurrent=0): -def upgrade_spock(cluster_name: str, new_spock_ver: str): + +# Regex that matches typical Spock‑5 version strings or component names +_SPOCK5_RE = re.compile(r"^(?:spock)?5[0-9\.]*$", re.IGNORECASE) + + +def upgrade_spock(): """ - Upgrade the Spock extension cluster-wide. + Validate Spock ↔ PostgreSQL compatibility **unless Spock 5 is already installed**. - ./pgedge um upgrade-spock + Behaviour + --------- + • If any installed Spock component matches `_SPOCK5_RE`, print a notice and EXIT 0. + • Otherwise, warn about downtime, read PG/Spock versions from SQLite, + do util.validate_spock_pg_compat(), and exit(1) on any error. + """ + DB_PATH = "data/conf/db_local.db" + VERSION_SQL = """ + SELECT pg.version AS pg_ver, + sp.version AS spock_ver, + sp.component AS spock_comp + FROM components AS pg + LEFT JOIN components AS sp + ON sp.component LIKE 'spock%' || pg.component + WHERE pg.component IN ('pg11','pg12','pg13','pg14','pg15','pg16','pg17') + LIMIT 1; """ - # 1. Warn that downgrades/same‐version upgrades aren’t supported - util.message( - "WARNING: Downgrading to a previous Spock version—or upgrading within the same major version—is not supported; " - "only upgrades to a newer major version are allowed.", - "warn", - isJSON, - ) - # 2. Load cluster JSON (db list, settings, and node definitions) - util.message(f"## Loading cluster '{cluster_name}' JSON definition", "info", isJSON) - try: - db_list, db_settings, nodes = load_json(cluster_name) - except Exception as e: - util.exit_message(f"Unable to load cluster JSON: {e}", 1, isJSON) - if not nodes: - util.exit_message("No 'nodes' found in cluster JSON.", 1, isJSON) - if not db_list: - util.exit_message("No 'databases' defined in cluster JSON.", 1, isJSON) - - # 3. Extract current Spock version - current_spock_ver = db_settings.get("spock_version") - if not current_spock_ver: - util.exit_message( - "Current Spock version not found in cluster JSON.", - 1, - isJSON - ) - util.message(f"Detected current Spock version from JSON: {current_spock_ver}", "info", isJSON) - - # 4. Validate requested upgrade - if new_spock_ver == current_spock_ver: - util.exit_message( - f"New Spock version {new_spock_ver} is the same as the current version.", - 1, - isJSON - ) + # ── fetch versions ──────────────────────────────────────────────────────────── try: - current_major = int(current_spock_ver.split('.')[0]) - new_major = int(new_spock_ver.split('.')[0]) - except (IndexError, ValueError): - util.exit_message( - f"Invalid version format: current='{current_spock_ver}', requested='{new_spock_ver}'.", - 1, - isJSON - ) - if new_major <= current_major: - util.exit_message( - f"Invalid upgrade: new Spock major version {new_major} must be greater than current {current_major}.", - 1, - isJSON - ) - - # 5. Gather node directories - node_dirs = [] - for n in nodes: - dir_ = n.get('path') - if dir_: - node_dirs.append(dir_) - if not node_dirs: - util.exit_message("Node directories not defined in JSON.", 1, isJSON) - util.message(f"Discovered node directories: {', '.join(node_dirs)}", "info", isJSON) - - # 6. Install new Spock major if needed - if new_major > current_major: - for base in node_dirs: - pgedge_dir = os.path.join(base, 'pgedge') - util.message( - f"Installing Spock{new_major} on node '{pgedge_dir}'", "info", isJSON - ) - try: - subprocess.run(['./pgedge', 'um', 'install', f'spock{new_major}'], cwd=pgedge_dir, check=True) - except subprocess.CalledProcessError as e: - util.exit_message(f"Spock install failed: {e}", 1, isJSON) - - # 7. Downtime warning - util.message( - f"WARNING: Upgrading Spock on '{cluster_name}' from {current_spock_ver} to {new_spock_ver} will cause downtime.", - "warn", - isJSON - ) + with _sqlite3.connect(DB_PATH) as conn: + row = conn.execute(VERSION_SQL).fetchone() + except _sqlite3.Error as err: + sys.exit(f"ERROR: SQLite query failed: {err}") - # 8. Backup with pgBackRest if configured - for n in nodes: - br = n.get('backrest') - if br: - stanza = br.get('stanza') - node_dir = n['path'] - pgedge_dir = os.path.join(node_dir, 'pgedge') - util.message(f"Running backup stanza '{stanza}'", "info", isJSON) - try: - subprocess.run(['./pgedge', 'backrest', 'backup', stanza], cwd=pgedge_dir, check=True) - except subprocess.CalledProcessError as e: - util.exit_message(f"Backup failed on node {n.get('name')}: {e}", 1, isJSON) - - # 9. Detect PostgreSQL version - try: - pg_version = db_settings.get('pg_version') or util.fetch_pg_version(node_dirs[0]) - util.message(f"Detected PostgreSQL version: {pg_version}", "info", isJSON) - except Exception as e: - util.exit_message(f"Could not detect PostgreSQL version: {e}", 1, isJSON) - - # 10. Compatibility check - util.validate_spock_pg_compat(new_spock_ver, pg_version) - - # 11. Perform ALTER EXTENSION for each db on each node - for n in nodes: - node_name = n.get('name') or n['path'] - node_dir = n['path'] - port = n.get('port') or db_settings.get('port') - if not port: - util.exit_message(f"Port missing for node '{node_name}'", 1, isJSON) - - # Build path to psql - pg_bin = f"pg{pg_version}" - psql_exec = os.path.join(node_dir, 'pgedge', pg_bin, 'bin', 'psql') - - - for db in db_list: - db_name = db.get('db_name') - db_user = db.get('db_user') - if not db_name or not db_user: - util.exit_message( - f"Database entry incomplete in JSON for node '{node_name}'", 1, isJSON - ) - util.message( - f"Altering Spock in {db_name}@{node_name} as {db_user}", - "info", - isJSON - ) - try: - subprocess.run( - [ - psql_exec, - '-p', str(port), - '-U', db_user, - '-d', db_name, - '-c', f"ALTER EXTENSION spock UPDATE TO \"{new_spock_ver}\";" - ], - check=True - ) - except subprocess.CalledProcessError as e: - util.exit_message(f"ALTER EXTENSION failed: {e}", 1, isJSON) - - # 12. Complete - util.exit_message( - f"Successfully upgraded Spock to {new_spock_ver} on cluster '{cluster_name}'.", - 0, - isJSON - ) + if not row: + sys.exit("ERROR: No PostgreSQL/Spock version row found.") + + pg_ver, spock_ver, spock_comp = row + # ── early‑exit if Spock 5 already installed ────────────────────────────────── + if any(_SPOCK5_RE.match(x or "") for x in (spock_ver, spock_comp)): + print(f"Spock 5 already installed (component='{spock_comp}', version='{spock_ver}').") + print("Nothing to do — skipping upgrade/installation.\n") + sys.exit(0) # treat as successful no‑op + + # ── standard downtime banner ───────────────────────────────────────────────── + banner = "=" * 80 + print(f"\n{banner}\n*** WARNING: This operation will cause downtime! ***\n{banner}\n") + print(f"Detected Spock version {spock_ver} on PostgreSQL {pg_ver}") + + # ── compatibility check ────────────────────────────────────────────────────── + try: + util.validate_spock_pg_compat(spock_ver, pg_ver) + except Exception as exc: + sys.exit(f"ERROR: Compatibility check failed: {exc}") + print("Compatibility check passed. Proceed with upgrade.") if __name__ == "__main__": fire.Fire( From e6b54c24631563a67acf3c4a2f7400193a15266d Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Mon, 21 Jul 2025 21:40:11 +0500 Subject: [PATCH 04/10] Update um.py Upgrade-spock function hide --- cli/scripts/um.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cli/scripts/um.py b/cli/scripts/um.py index 52b12ff9..62cc315d 100755 --- a/cli/scripts/um.py +++ b/cli/scripts/um.py @@ -3,8 +3,6 @@ import os, sys, glob, sqlite3, time import fire, meta, util -from cluster import load_json, get_cluster_json # helper functions for reading cluster JSON -import subprocess isJSON = util.isJSON import re import sqlite3 as _sqlite3 @@ -315,6 +313,5 @@ def upgrade_spock(): "clean": clean, "verify-metadata": verify_metadata, "download": download, - "upgrade-spock": upgrade_spock, } ) From b9d96e930ccad6c0bfb9de05c64985a2bb8bea0e Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Tue, 22 Jul 2025 16:10:50 +0500 Subject: [PATCH 05/10] code refactored --- cli/scripts/um.py | 89 +++++++++++++++++++++++------------------------ 1 file changed, 43 insertions(+), 46 deletions(-) diff --git a/cli/scripts/um.py b/cli/scripts/um.py index 62cc315d..cacac938 100755 --- a/cli/scripts/um.py +++ b/cli/scripts/um.py @@ -85,26 +85,29 @@ def update(): +# Match both "spock50" and "spock50-pg17" (and variants like spock5, spock5-pg16) +_SPOCK5_NAME_RE = re.compile(r"^spock5(?:0)?(?:-pg\d+)?$", re.IGNORECASE) +_SPOCK5_VER_RE = re.compile(r"^5\.", re.IGNORECASE) +_SPOCK50_RE = re.compile(r"^spock50(?:-pg\d+)?$", re.IGNORECASE) + def install(component, active=True): - """Install a component or trigger Spock upgrade for spock50-pg.""" - # Only invoke upgrade_spock when installing spock50-pg* - if component.startswith("spock50-pg"): - print(f"Detected Spock upgrade target '{component}', invoking upgrade_spock()...\n") - upgrade_spock() - # After successful compatibility, proceed with install - util.message(f"um.install(install {component})", "debug") - run_cmd("install", component) - return + """Install a component.""" + + # Trigger pre-check ONLY for Spock 5.0 artifacts + if _SPOCK50_RE.match(component): + print(f"Detected Spock 5 target '{component}', running upgrade_spock()...\n") + validate_spock_upgrade() # should sys.exit(1) on failure; otherwise just returns + # Common path (no duplication) if active not in (True, False): util.exit_message("'active' parm must be True or False") - cmd = "install" - if not active: - cmd = "install --no-preload" + + cmd = "install" if active else "install --no-preload" util.message(f"um.install({cmd} {component})", "debug") run_cmd(cmd, component) + def remove(component): """Uninstall a component.""" installed_comp_list = meta.get_component_list() @@ -242,36 +245,27 @@ def verify_metadata(Project="", Stage="prod", IsCurrent=0): meta.pretty_sql(sql) - - - -# Regex that matches typical Spock‑5 version strings or component names -_SPOCK5_RE = re.compile(r"^(?:spock)?5[0-9\.]*$", re.IGNORECASE) - - -def upgrade_spock(): +def validate_spock_upgrade(): """ - Validate Spock ↔ PostgreSQL compatibility **unless Spock 5 is already installed**. + Validate Spock↔PostgreSQL compatibility for an upcoming Spock 5 install. - Behaviour - --------- - • If any installed Spock component matches `_SPOCK5_RE`, print a notice and EXIT 0. - • Otherwise, warn about downtime, read PG/Spock versions from SQLite, - do util.validate_spock_pg_compat(), and exit(1) on any error. + • If Spock 5 already installed (name or version), print notice and return 0. + • Else: warn about downtime, load versions from SQLite, call util.validate_spock_pg_compat(). + Exit(1) on failure, return 0 on success. """ DB_PATH = "data/conf/db_local.db" VERSION_SQL = """ - SELECT pg.version AS pg_ver, - sp.version AS spock_ver, - sp.component AS spock_comp - FROM components AS pg - LEFT JOIN components AS sp - ON sp.component LIKE 'spock%' || pg.component - WHERE pg.component IN ('pg11','pg12','pg13','pg14','pg15','pg16','pg17') + SELECT pg.version AS pg_ver, + sp.version AS spock_ver, + sp.component AS spock_comp + FROM components AS pg + LEFT JOIN components AS sp + ON sp.component LIKE 'spock%' || pg.component + WHERE pg.component LIKE 'pg__' + ORDER BY CAST(substr(pg.component, 3) AS INTEGER) DESC LIMIT 1; - """ + """ - # ── fetch versions ──────────────────────────────────────────────────────────── try: with _sqlite3.connect(DB_PATH) as conn: row = conn.execute(VERSION_SQL).fetchone() @@ -283,24 +277,27 @@ def upgrade_spock(): pg_ver, spock_ver, spock_comp = row - # ── early‑exit if Spock 5 already installed ────────────────────────────────── - if any(_SPOCK5_RE.match(x or "") for x in (spock_ver, spock_comp)): - print(f"Spock 5 already installed (component='{spock_comp}', version='{spock_ver}').") - print("Nothing to do — skipping upgrade/installation.\n") - sys.exit(0) # treat as successful no‑op + # Already on Spock 5? No-op. + if _SPOCK5_NAME_RE.match(spock_comp or "") or _SPOCK5_VER_RE.match(spock_ver or ""): + print(f"Spock 5 already installed (component='{spock_comp}', version='{spock_ver}').") + print("Skipping upgrade_spock checks.\n") + return 0 - # ── standard downtime banner ───────────────────────────────────────────────── + # Downtime warning banner = "=" * 80 - print(f"\n{banner}\n*** WARNING: This operation will cause downtime! ***\n{banner}\n") - print(f"Detected Spock version {spock_ver} on PostgreSQL {pg_ver}") + print(f"\n{banner}") + print("*** WARNING: This operation will cause downtime! ***") + print(f"{banner}\n") + print(f"Detected Spock version {spock_ver or 'N/A'} on PostgreSQL {pg_ver}") - # ── compatibility check ────────────────────────────────────────────────────── + # Compatibility check try: util.validate_spock_pg_compat(spock_ver, pg_ver) except Exception as exc: sys.exit(f"ERROR: Compatibility check failed: {exc}") - print("Compatibility check passed. Proceed with upgrade.") + print("Compatibility check passed.") + return 0 if __name__ == "__main__": fire.Fire( @@ -312,6 +309,6 @@ def upgrade_spock(): "upgrade": upgrade, "clean": clean, "verify-metadata": verify_metadata, - "download": download, + "download": download, } ) From e3cebf4f0535944e1fd69d34e7cb569d76b031e9 Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Wed, 23 Jul 2025 21:28:24 +0500 Subject: [PATCH 06/10] additional test case added for spock 5 upgrade comp check --- cli/scripts/um.py | 78 +++++++++++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 26 deletions(-) diff --git a/cli/scripts/um.py b/cli/scripts/um.py index cacac938..f860106a 100755 --- a/cli/scripts/um.py +++ b/cli/scripts/um.py @@ -248,37 +248,48 @@ def verify_metadata(Project="", Stage="prod", IsCurrent=0): def validate_spock_upgrade(): """ Validate Spock↔PostgreSQL compatibility for an upcoming Spock 5 install. - - • If Spock 5 already installed (name or version), print notice and return 0. - • Else: warn about downtime, load versions from SQLite, call util.validate_spock_pg_compat(). - Exit(1) on failure, return 0 on success. """ + DB_PATH = "data/conf/db_local.db" - VERSION_SQL = """ - SELECT pg.version AS pg_ver, - sp.version AS spock_ver, - sp.component AS spock_comp - FROM components AS pg - LEFT JOIN components AS sp - ON sp.component LIKE 'spock%' || pg.component - WHERE pg.component LIKE 'pg__' - ORDER BY CAST(substr(pg.component, 3) AS INTEGER) DESC - LIMIT 1; - """ + + SPOCK_SQL = """ + SELECT version AS spock_ver, component AS spock_comp + FROM components + WHERE component LIKE 'spock%' + ORDER BY CASE + WHEN version LIKE '5.%' THEN 5 + WHEN version LIKE '4.%' THEN 4 + ELSE 0 + END DESC, + version DESC + LIMIT 1; + """ + + PG_SQL = """ + SELECT version AS pg_ver + FROM components + WHERE component LIKE 'pg__' + ORDER BY CAST(substr(component, 3) AS INTEGER) DESC + LIMIT 1; + """ try: with _sqlite3.connect(DB_PATH) as conn: - row = conn.execute(VERSION_SQL).fetchone() + sp_row = conn.execute(SPOCK_SQL).fetchone() + pg_row = conn.execute(PG_SQL).fetchone() except _sqlite3.Error as err: sys.exit(f"ERROR: SQLite query failed: {err}") - if not row: - sys.exit("ERROR: No PostgreSQL/Spock version row found.") + if not pg_row: + sys.exit("ERROR: No PostgreSQL version row found.") - pg_ver, spock_ver, spock_comp = row + pg_ver = pg_row[0] + spock_ver, spock_comp = (sp_row or (None, None)) # Already on Spock 5? No-op. - if _SPOCK5_NAME_RE.match(spock_comp or "") or _SPOCK5_VER_RE.match(spock_ver or ""): + if spock_ver and ( + _SPOCK5_NAME_RE.match(spock_comp) or _SPOCK5_VER_RE.match(spock_ver) + ): print(f"Spock 5 already installed (component='{spock_comp}', version='{spock_ver}').") print("Skipping upgrade_spock checks.\n") return 0 @@ -288,13 +299,28 @@ def validate_spock_upgrade(): print(f"\n{banner}") print("*** WARNING: This operation will cause downtime! ***") print(f"{banner}\n") - print(f"Detected Spock version {spock_ver or 'N/A'} on PostgreSQL {pg_ver}") - # Compatibility check - try: - util.validate_spock_pg_compat(spock_ver, pg_ver) - except Exception as exc: - sys.exit(f"ERROR: Compatibility check failed: {exc}") + # --- Three cases --- + if not spock_ver: + # Case 1: no Spock installed + print(f"Detected PostgreSQL version {pg_ver} (no Spock version found)") + try: + util.validate_spock_pg_compat('50', pg_ver) + except Exception as exc: + sys.exit(f"ERROR: Compatibility check failed: {exc}") + + elif spock_ver.startswith('4'): + # Case 2: Spock 4.x installed + print(f"Detected Spock version {spock_ver} on PostgreSQL {pg_ver}") + try: + util.validate_spock_pg_compat(spock_ver, pg_ver) + except Exception as exc: + sys.exit(f"ERROR: Compatibility check failed: {exc}") + + else: + # Case 3: Spock 5.x installed + print(f"Spock version {spock_ver} detected; skipping compatibility check.") + return 0 print("Compatibility check passed.") return 0 From 31435ea8116feeab55cec40b3817b1470361318c Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Wed, 23 Jul 2025 21:35:05 +0500 Subject: [PATCH 07/10] additional test case added for spock 5 upgrade comp check --- cli/scripts/um.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cli/scripts/um.py b/cli/scripts/um.py index f860106a..40391b64 100755 --- a/cli/scripts/um.py +++ b/cli/scripts/um.py @@ -95,7 +95,7 @@ def install(component, active=True): # Trigger pre-check ONLY for Spock 5.0 artifacts if _SPOCK50_RE.match(component): - print(f"Detected Spock 5 target '{component}', running upgrade_spock()...\n") + print(f"Detected Spock 5 target '{component}'") validate_spock_upgrade() # should sys.exit(1) on failure; otherwise just returns # Common path (no duplication) @@ -291,7 +291,6 @@ def validate_spock_upgrade(): _SPOCK5_NAME_RE.match(spock_comp) or _SPOCK5_VER_RE.match(spock_ver) ): print(f"Spock 5 already installed (component='{spock_comp}', version='{spock_ver}').") - print("Skipping upgrade_spock checks.\n") return 0 # Downtime warning @@ -303,7 +302,7 @@ def validate_spock_upgrade(): # --- Three cases --- if not spock_ver: # Case 1: no Spock installed - print(f"Detected PostgreSQL version {pg_ver} (no Spock version found)") + print(f"Detected PostgreSQL version {pg_ver} ") try: util.validate_spock_pg_compat('50', pg_ver) except Exception as exc: From cd96d21403ebfa2d51604a85efa527cc6dc70f1e Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Wed, 23 Jul 2025 21:49:54 +0500 Subject: [PATCH 08/10] additional test case added for spock 5 upgrade comp check --- cli/scripts/um.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cli/scripts/um.py b/cli/scripts/um.py index 40391b64..2a241a2f 100755 --- a/cli/scripts/um.py +++ b/cli/scripts/um.py @@ -95,7 +95,6 @@ def install(component, active=True): # Trigger pre-check ONLY for Spock 5.0 artifacts if _SPOCK50_RE.match(component): - print(f"Detected Spock 5 target '{component}'") validate_spock_upgrade() # should sys.exit(1) on failure; otherwise just returns # Common path (no duplication) @@ -290,8 +289,7 @@ def validate_spock_upgrade(): if spock_ver and ( _SPOCK5_NAME_RE.match(spock_comp) or _SPOCK5_VER_RE.match(spock_ver) ): - print(f"Spock 5 already installed (component='{spock_comp}', version='{spock_ver}').") - return 0 + return 0 # Downtime warning banner = "=" * 80 @@ -302,7 +300,7 @@ def validate_spock_upgrade(): # --- Three cases --- if not spock_ver: # Case 1: no Spock installed - print(f"Detected PostgreSQL version {pg_ver} ") + try: util.validate_spock_pg_compat('50', pg_ver) except Exception as exc: From a2a837032b5c994d7d37e414234be3c37fb18c9f Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Wed, 23 Jul 2025 22:53:16 +0500 Subject: [PATCH 09/10] warning banner removed from case 3 --- cli/scripts/um.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cli/scripts/um.py b/cli/scripts/um.py index 2a241a2f..36fd70ee 100755 --- a/cli/scripts/um.py +++ b/cli/scripts/um.py @@ -293,14 +293,13 @@ def validate_spock_upgrade(): # Downtime warning banner = "=" * 80 - print(f"\n{banner}") - print("*** WARNING: This operation will cause downtime! ***") - print(f"{banner}\n") # --- Three cases --- if not spock_ver: # Case 1: no Spock installed - + print(f"\n{banner}") + print("*** WARNING: This operation will cause downtime! ***") + print(f"{banner}\n") try: util.validate_spock_pg_compat('50', pg_ver) except Exception as exc: @@ -308,6 +307,9 @@ def validate_spock_upgrade(): elif spock_ver.startswith('4'): # Case 2: Spock 4.x installed + print(f"\n{banner}") + print("*** WARNING: This operation will cause downtime! ***") + print(f"{banner}\n") print(f"Detected Spock version {spock_ver} on PostgreSQL {pg_ver}") try: util.validate_spock_pg_compat(spock_ver, pg_ver) From dccd7f613837084554c2cf36d32776192a0c8501 Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Wed, 23 Jul 2025 15:21:58 -0500 Subject: [PATCH 10/10] final adjustment to logging output --- cli/scripts/um.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/cli/scripts/um.py b/cli/scripts/um.py index 36fd70ee..ceff8e3a 100755 --- a/cli/scripts/um.py +++ b/cli/scripts/um.py @@ -294,19 +294,15 @@ def validate_spock_upgrade(): # Downtime warning banner = "=" * 80 - # --- Three cases --- if not spock_ver: - # Case 1: no Spock installed - print(f"\n{banner}") - print("*** WARNING: This operation will cause downtime! ***") - print(f"{banner}\n") + # Case 1: no existing Spock installed, run validation but don't print banner try: util.validate_spock_pg_compat('50', pg_ver) except Exception as exc: sys.exit(f"ERROR: Compatibility check failed: {exc}") elif spock_ver.startswith('4'): - # Case 2: Spock 4.x installed + # Case 2: Spock 4.x installed, this is a major upgrade print(f"\n{banner}") print("*** WARNING: This operation will cause downtime! ***") print(f"{banner}\n") @@ -316,12 +312,7 @@ def validate_spock_upgrade(): except Exception as exc: sys.exit(f"ERROR: Compatibility check failed: {exc}") - else: - # Case 3: Spock 5.x installed - print(f"Spock version {spock_ver} detected; skipping compatibility check.") - return 0 - - print("Compatibility check passed.") + print("Compatibility check passed.") return 0 if __name__ == "__main__":