From dbb68fabdb19953b136d1d62b9acfe91265256c9 Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Mon, 30 Jun 2025 15:38:39 +0500 Subject: [PATCH 01/25] Spock 5 integration --- cli/scripts/cluster.py | 5 +- cli/scripts/meta.py | 1 - cli/scripts/setup.py | 3 +- cli/scripts/util.py | 120 ++++++++++++++++++++++++++++++++++++++++- src/conf/versions.sql | 22 ++------ 5 files changed, 127 insertions(+), 24 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index a4915c4a..8f3bf6db 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -6,6 +6,7 @@ import meta import time import sys +import setup_core import getpass from tabulate import tabulate # type: ignore from ipaddress import ip_address @@ -1312,7 +1313,9 @@ def init(cluster_name, install=True): parsed_json = get_cluster_json(cluster_name) if parsed_json is None: util.exit_message("Unable to load cluster JSON", 1) - + pg_version = db_settings["pg_version"] + spock_ver = db_settings.get("spock_version", util.get_default_spock(pg_version)) + util.validate_spock_pg_compat(spock_ver, pg_version) verbose = parsed_json.get("log_level", "info") all_nodes = nodes.copy() diff --git a/cli/scripts/meta.py b/cli/scripts/meta.py index b399e43a..9cd5fb1c 100644 --- a/cli/scripts/meta.py +++ b/cli/scripts/meta.py @@ -306,7 +306,6 @@ def get_default_spock(pgv): + pgv + "' \n" + " AND component LIKE 'spock%'" - + " AND version not LIKE '%devel%'" ) try: c = con.cursor() diff --git a/cli/scripts/setup.py b/cli/scripts/setup.py index 2fc21aa9..6e5452d8 100755 --- a/cli/scripts/setup.py +++ b/cli/scripts/setup.py @@ -95,7 +95,8 @@ def setup_pgedge(User=None, Passwd=None, dbName=None, port=None, pg_data=None, p pg_ver = df_pg pg_major, pg_minor = setup_core.parse_pg(pg_ver) - + # Validate that if Spock ≥5 we're using PG 15.13+, 16.9+ or 17.5+Add commentMore actions + util.validate_spock_pg_compat(spock_ver, pg_ver) pg_init_options = "" if pg_data is not None: pg_data = pg_data.rstrip("/") diff --git a/cli/scripts/util.py b/cli/scripts/util.py index b4335a21..4a0ef6ea 100644 --- a/cli/scripts/util.py +++ b/cli/scripts/util.py @@ -8,8 +8,8 @@ MY_CODENAME = "" DEFAULT_PG = "16" -DEFAULT_SPOCK = "40" -DEFAULT_SPOCK_17 = "40" +DEFAULT_SPOCK = "50" +DEFAULT_SPOCK_17 = "50" MY_CMD = os.getenv("MY_CMD", None) MY_HOME = os.getenv("MY_HOME", None) MY_LIBS = f"{MY_HOME}/hub/scripts/lib" @@ -37,6 +37,7 @@ import subprocess import getpass import filecmp +import re from subprocess import Popen, PIPE, STDOUT from datetime import datetime, timedelta from urllib import request as urllib2 @@ -123,7 +124,122 @@ def get_default_spock(pgv): return(DEFAULT_SPOCK) +def get_default_spock(pgv): + if pgv == "17": + return(DEFAULT_SPOCK_17) + + return(DEFAULT_SPOCK) + + + +def validate_spock_pg_compat(spock_ver: str = None, pg_ver: str = None) -> None: + """ + Compatibility rules: + • If Spock < 5.0.0 ⇒ works with any supported PostgreSQL major. + • If Spock ≥ 5.0.0 ⇒ + – PG15 must be ≥ 15.13 + – PG16 must be ≥ 16.9 + – PG17 must be ≥ 17.5 + + Also supports shorthand Spock strings: + – "50" → "5.0.0", "40" → "4.0.0", etc. + """ + # 0) Fill in defaults if user didn’t pass anything + if not pg_ver: + pg_ver = DEFAULT_PG + if not spock_ver: + maj = int(pg_ver.split(".", 1)[0]) + spock_ver = DEFAULT_SPOCK_17 if maj == 17 else DEFAULT_SPOCK + + # 0.5) Normalize two-digit shorthand (e.g. "50" → "5.0.0") + m = re.fullmatch(r'(\d)(\d)$', spock_ver) + if m: + spock_ver = f"{int(m.group(1))}.{int(m.group(2))}.0" + + # 1) Parse Spock version (abort on bad format) + try: + spv = Version(spock_ver) + except ValueError: + exit_message(f"Invalid Spock version '{spock_ver}'. Aborting.", 1, isJSON) + + # 2) If Spock < 5 ⇒ compatible with any PG + if spv.major < 5: + return + # — New block: handle pg_ver with “-1” or “-2” suffix + rev = None + rev_match = re.fullmatch(r'(\d+)\.(\d+)-(1|2)$', pg_ver) + if rev_match: + pg_major = int(rev_match.group(1)) + pg_patch = int(rev_match.group(2)) + rev = int(rev_match.group(3)) + + # reject revision “-1” on Spock ≥5 + if rev == 1: + exit_message( + f"Error: PostgreSQL {pg_major}.{pg_patch}-1 is not supported with Spock {spv}; " + "please use the “-2” revision instead.", + 1, + isJSON + ) + # for “-2”, we strip suffix and proceed with pg_major/pg_patch below + # end new block + + # 3) Spock ≥ 5 ⇒ enforce minimum‐patch for each PG major + minimum_patches = { + 15: 13, + 16: 9, + 17: 5, + } + + # 4) Extract PG major and patch (if not already set by rev_match) + if rev_match: + # pg_major, pg_patch are already set + pass + elif "." not in pg_ver: + # bare-major → use its minimum patch + try: + pg_major = int(pg_ver) + except ValueError: + exit_message(f"Invalid PostgreSQL version '{pg_ver}'. Aborting.", 1, isJSON) + if pg_major not in minimum_patches: + allowed = ", ".join(str(m) for m in minimum_patches) + exit_message( + f"Error: Spock {spv} supports only PostgreSQL majors {allowed}; " + f"you have {pg_major}. Aborting.", + 1, + isJSON + ) + pg_patch = minimum_patches[pg_major] + else: + parts = pg_ver.split(".", 2) + if len(parts) < 2: + exit_message(f"Invalid PostgreSQL version '{pg_ver}'. Aborting.", 1, isJSON) + try: + pg_major = int(parts[0]) + pg_patch = int(parts[1]) + except ValueError: + exit_message(f"Invalid PostgreSQL version '{pg_ver}'. Aborting.", 1, isJSON) + + # 5) Major must be supported + if pg_major not in minimum_patches: + allowed = ", ".join(str(m) for m in minimum_patches) + exit_message( + f"Error: Spock {spv} supports only PostgreSQL majors {allowed}; " + f"you have {pg_major}. Aborting.", + 1, + isJSON + ) + + # 6) Enforce minimum‐patch + required = minimum_patches[pg_major] + if pg_patch < required: + exit_message( + f"Error: Spock {spv} requires PostgreSQL {pg_major}.{required} or newer; " + f"you have {pg_major}.{pg_patch}. Aborting.", + 1, + isJSON + ) def get_cpu_info(): try: import cpuinfo diff --git a/src/conf/versions.sql b/src/conf/versions.sql index c6218575..f10e9127 100644 --- a/src/conf/versions.sql +++ b/src/conf/versions.sql @@ -74,11 +74,6 @@ CREATE TABLE extensions ( preload_name TEXT NOT NULL, default_conf TEXT NOT NULL ); -INSERT INTO extensions VALUES ('spock33', 'spock', 1, 'spock', - 'wal_level=logical | max_worker_processes=12 | max_replication_slots=16 | - max_wal_senders=16 | hot_standby_feedback=on | wal_sender_timeout=5s | - track_commit_timestamp=on | spock.conflict_resolution=last_update_wins | - spock.save_resolutions=on | spock.conflict_log_level=DEBUG'); INSERT INTO extensions VALUES ('spock40', 'spock', 1, 'spock', 'wal_level=logical | max_worker_processes=12 | max_replication_slots=16 | max_wal_senders=16 | hot_standby_feedback=on | wal_sender_timeout=5s | @@ -320,17 +315,6 @@ INSERT INTO versions VALUES ('snowflake-pg17', '2.2-1', 'amd, arm', 1, '20240626 -- ## SPOCK (parent project) ############ INSERT INTO projects VALUES ('spock', 'pge', 4, 0, '', 1, 'https://github.com/pgedge/spock/tags', 'spock', 1, 'spock.png', 'Logical Rep w/ Conflict Resolution', 'https://github.com/pgedge/spock/', 'pg_spock, pgsspock, vulcan'); - --- ## SPOCK33 ########################### -INSERT INTO releases VALUES ('spock33-pg15', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); -INSERT INTO releases VALUES ('spock33-pg16', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); - -INSERT INTO versions VALUES ('spock33-pg15', '3.3.6-1', 'amd, arm', 1, '20240820', 'pg15', '', ''); -INSERT INTO versions VALUES ('spock33-pg16', '3.3.6-1', 'amd, arm', 1, '20240820', 'pg16', '', ''); - -INSERT INTO versions VALUES ('spock33-pg15', '3.3.5-1', 'amd, arm', 0, '20240607', 'pg15', '', ''); -INSERT INTO versions VALUES ('spock33-pg16', '3.3.5-1', 'amd, arm', 0, '20240607', 'pg16', '', ''); - -- ## SPOCK40 ########################### INSERT INTO releases VALUES ('spock40-pg15', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); INSERT INTO releases VALUES ('spock40-pg16', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); @@ -350,9 +334,9 @@ INSERT INTO versions VALUES ('spock40-pg17', '4.0.8-1', 'amd, arm', 0, '20241218 -- ## spock50 ########################### -INSERT INTO releases VALUES ('spock50-pg15', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); -INSERT INTO releases VALUES ('spock50-pg16', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); -INSERT INTO releases VALUES ('spock50-pg17', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); +INSERT INTO releases VALUES ('spock50-pg15', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); +INSERT INTO releases VALUES ('spock50-pg16', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); +INSERT INTO releases VALUES ('spock50-pg17', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); INSERT INTO versions VALUES ('spock50-pg15', '5.0.0-1', 'amd, arm', 1, '20250715', 'pg15', '', ''); INSERT INTO versions VALUES ('spock50-pg16', '5.0.0-1', 'amd, arm', 1, '20250715', 'pg16', '', ''); From 1ee7cf207d0634c19aa71ba565922969b2f86903 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Mon, 30 Jun 2025 20:21:03 +0500 Subject: [PATCH 02/25] function duplication removed from util.py --- cli/scripts/util.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cli/scripts/util.py b/cli/scripts/util.py index 4a0ef6ea..22f1e5f4 100644 --- a/cli/scripts/util.py +++ b/cli/scripts/util.py @@ -124,12 +124,6 @@ def get_default_spock(pgv): return(DEFAULT_SPOCK) -def get_default_spock(pgv): - if pgv == "17": - return(DEFAULT_SPOCK_17) - - return(DEFAULT_SPOCK) - def validate_spock_pg_compat(spock_ver: str = None, pg_ver: str = None) -> None: From e469e4f8ff5a7675deb9b556ea211e5eba3b5471 Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Mon, 30 Jun 2025 16:37:30 -0500 Subject: [PATCH 03/25] support --rm-data if custom data directory is in use (#340) --- src/pgXX/remove-pgXX.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pgXX/remove-pgXX.py b/src/pgXX/remove-pgXX.py index 28203cff..fcd77ce9 100644 --- a/src/pgXX/remove-pgXX.py +++ b/src/pgXX/remove-pgXX.py @@ -15,5 +15,6 @@ isRM_DATA = os.getenv("isRM_DATA", "False") if isRM_DATA == "True": util.message("Removing 'data' directories at your request") - util.echo_cmd(f"sudo rm -r data/{pgver}") + data_dir = util.get_column("datadir", pgver) + util.echo_cmd(f"sudo rm -r {data_dir}") util.echo_cmd(f"sudo rm -r data/logs/{pgver}") From 7fba7aa93b4412ea70ce8a3dd04253400e8153df Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Tue, 1 Jul 2025 20:42:51 +0500 Subject: [PATCH 04/25] Pg17 is default --- cli/scripts/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/scripts/util.py b/cli/scripts/util.py index 22f1e5f4..273f1134 100644 --- a/cli/scripts/util.py +++ b/cli/scripts/util.py @@ -7,7 +7,7 @@ MY_VERSION = "25.1.0" MY_CODENAME = "" -DEFAULT_PG = "16" +DEFAULT_PG = "17" DEFAULT_SPOCK = "50" DEFAULT_SPOCK_17 = "50" MY_CMD = os.getenv("MY_CMD", None) From b1130381c0903e83eb276fd54390fa0921a67b05 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Tue, 1 Jul 2025 20:43:14 +0500 Subject: [PATCH 05/25] build_all.sh default pg17 --- build_all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_all.sh b/build_all.sh index f584fbdf..3f8f5f22 100755 --- a/build_all.sh +++ b/build_all.sh @@ -13,7 +13,7 @@ if [ ! $num_p == "0" ] && [ ! $num_p == "1" ]; then fi if [ "$1" == "" ]; then - majorV=16 + majorV=17 echo "" echo "### Defaulting to pg $majorV ###" else From a62e5387eded29317a1e8abb782b5964cc39aa01 Mon Sep 17 00:00:00 2001 From: Gabrielle Poncey Date: Tue, 1 Jul 2025 11:06:36 -0700 Subject: [PATCH 06/25] fix: db guc set supports quoted params\n Adds an extra set of quotations to the guc_value portion of the ALTER SYSTEM SET SQL in order to support spaced, quoted guc values \n PLAT-44 --- cli/scripts/db.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cli/scripts/db.py b/cli/scripts/db.py index f5138bcc..3174a389 100755 --- a/cli/scripts/db.py +++ b/cli/scripts/db.py @@ -109,18 +109,20 @@ def create(db=None, User=None, Passwd=None, pg=None, spock=None, help=False): def guc_set(guc_name, guc_value): """Set GUC.""" - pg_v, spock_v = util.get_pg_v() pg = pg_v[2:] nc = "./pgedge " ncb = nc + "pgbin " + str(pg) + " " - cmd = f"ALTER SYSTEM SET {guc_name} = {guc_value}" + cmd = f"ALTER SYSTEM SET {guc_name} = '{guc_value}'" + rc1 = util.echo_cmd(ncb + '"psql -q -c \\"' + cmd + '\\" postgres"',False) cmd = f"SELECT pg_reload_conf()" rc2 = util.echo_cmd(ncb + '"psql -q -c \\"' + cmd + '\\" postgres"',False) + rcs = rc1 + rc2 + if rcs == 0: util.message(f"Set GUC {guc_name} to {guc_value}","info") else: From 8826078a954646ece830ac8a8d429e48439fdb2e Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Wed, 2 Jul 2025 18:57:07 +0500 Subject: [PATCH 07/25] spock5 issue with regression test --- cli/scripts/util.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cli/scripts/util.py b/cli/scripts/util.py index 273f1134..a8dc0ab2 100644 --- a/cli/scripts/util.py +++ b/cli/scripts/util.py @@ -138,12 +138,18 @@ def validate_spock_pg_compat(spock_ver: str = None, pg_ver: str = None) -> None: Also supports shorthand Spock strings: – "50" → "5.0.0", "40" → "4.0.0", etc. """ - # 0) Fill in defaults if user didn’t pass anything + # 0) Fill in defaults if user didn’t pass anything if not pg_ver: pg_ver = DEFAULT_PG + else: + pg_ver = str(pg_ver) # ← force to string + if not spock_ver: maj = int(pg_ver.split(".", 1)[0]) spock_ver = DEFAULT_SPOCK_17 if maj == 17 else DEFAULT_SPOCK + else: + spock_ver = str(spock_ver) # ← force to string + # 0.5) Normalize two-digit shorthand (e.g. "50" → "5.0.0") m = re.fullmatch(r'(\d)(\d)$', spock_ver) From 7ec23abaaf111925081af1cc3d60c1fc6ce562fd Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Wed, 2 Jul 2025 23:13:39 +0500 Subject: [PATCH 08/25] pg1-host issue resolved --- src/backrest/backrest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backrest/backrest.py b/src/backrest/backrest.py index 2a805509..997742fe 100755 --- a/src/backrest/backrest.py +++ b/src/backrest/backrest.py @@ -30,8 +30,7 @@ def fetch_config(): "repo1-retention-full-type", "repo1-path", "repo1-host-user", "repo1-host", "repo1-cipher-type", "log-level-console", "repo1-type", "process-max", "compress-level", "pg1-path", - "pg1-user", "pg1-database", "db-socket-path", "pg1-port", - "pg1-host" + "pg1-user", "pg1-database", "db-socket-path", "pg1-port" ] for param in params: config[param] = util.get_value("BACKUP", param) From ecc8a232aee8c053176127f60c233df4b6ac89e4 Mon Sep 17 00:00:00 2001 From: Gabrielle Poncey Date: Thu, 3 Jul 2025 10:00:27 -0700 Subject: [PATCH 09/25] fix: db guc set supports quoted params Adds an extra set of quotations to the guc_value and escapes / or quotations it may hold to allow for spaced and quoted parameters to be passed PLAT-44 --- cli/scripts/db.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/scripts/db.py b/cli/scripts/db.py index 3174a389..ae1e57d2 100755 --- a/cli/scripts/db.py +++ b/cli/scripts/db.py @@ -8,6 +8,7 @@ import json import util import fire +import re def create(db=None, User=None, Passwd=None, pg=None, spock=None, help=False): """ @@ -109,12 +110,14 @@ def create(db=None, User=None, Passwd=None, pg=None, spock=None, help=False): def guc_set(guc_name, guc_value): """Set GUC.""" + pg_v, spock_v = util.get_pg_v() pg = pg_v[2:] nc = "./pgedge " ncb = nc + "pgbin " + str(pg) + " " + guc_value = re.sub(r'([\'\\])', r'\1\1', str(guc_value)) cmd = f"ALTER SYSTEM SET {guc_name} = '{guc_value}'" rc1 = util.echo_cmd(ncb + '"psql -q -c \\"' + cmd + '\\" postgres"',False) From 6a345068c02c45557c21c91c647adbe734f8933c Mon Sep 17 00:00:00 2001 From: Gabrielle Poncey Date: Thu, 3 Jul 2025 11:35:05 -0700 Subject: [PATCH 10/25] json-create supports hostnames node config json-create allows creation of a node or subnode with either ip address or hostname through the use of socket.gethostbyname PLAT-131 --- cli/scripts/cluster.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index 8f3bf6db..f479165c 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -9,10 +9,11 @@ import setup_core import getpass from tabulate import tabulate # type: ignore -from ipaddress import ip_address +import socket import os import re + BASE_DIR = "cluster" DEFAULT_REPO = "https://pgedge-download.s3.amazonaws.com/REPO" @@ -1064,9 +1065,11 @@ def get_cluster_info(cluster_name): ) try: if public_ip: - ip_address(public_ip) + socket.gethostbyname(public_ip) + if private_ip: - ip_address(private_ip) + socket.gethostbyname(private_ip) + except ValueError: validation_errors.append( f"Invalid IP address provided for node {node.get('name')}." @@ -1099,9 +1102,9 @@ def get_cluster_info(cluster_name): ) try: if public_ip: - ip_address(public_ip) + socket.gethostbyname(public_ip) if private_ip: - ip_address(private_ip) + socket.gethostbyname(private_ip) except ValueError: validation_errors.append( f"Invalid IP address provided for sub-node {sub_node.get('name')}." From f3239e2ea63b489d9f48f60836abe613f1a56719 Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Tue, 8 Jul 2025 16:24:22 +0500 Subject: [PATCH 11/25] pg1-host remove from install-backrest module --- src/backrest/install-backrest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backrest/install-backrest.py b/src/backrest/install-backrest.py index 461168d1..e9ffef32 100644 --- a/src/backrest/install-backrest.py +++ b/src/backrest/install-backrest.py @@ -58,7 +58,6 @@ def configure_backup_settings(): "pg1-path": "xx", "pg1-user": "xx", "pg1-port": "5432", - "pg1-host": "127.0.0.1", "db-socket-path": "/tmp", "global:archive-push": { "compress-level": "3" From f0f0ae88adaa82150368a17e448f285c894a111e Mon Sep 17 00:00:00 2001 From: Gabrielle Poncey Date: Tue, 8 Jul 2025 10:00:55 -0700 Subject: [PATCH 12/25] fix: appropriate error caught invalid host/ip In the case that a hostname or ip address cannot be resolved by gethostbyname, socket error will be caught, appended to validation_errors and reported with the specific node from which it arose from. PLAT-131 --- cli/scripts/cluster.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index f479165c..766abe39 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -1070,9 +1070,9 @@ def get_cluster_info(cluster_name): if private_ip: socket.gethostbyname(private_ip) - except ValueError: + except socket.gaierror as e: validation_errors.append( - f"Invalid IP address provided for node {node.get('name')}." + f"Error resolving hostname or ip adress for node {node.get('name')} : {e}." ) for sub_node in node.get("sub_nodes", []): @@ -1105,9 +1105,9 @@ def get_cluster_info(cluster_name): socket.gethostbyname(public_ip) if private_ip: socket.gethostbyname(private_ip) - except ValueError: + except socket.gaierror as e: validation_errors.append( - f"Invalid IP address provided for sub-node {sub_node.get('name')}." + f"Error resolving hostname or ip adress for sub-node {sub_node.get('name')}: {e}." ) if validation_errors: From 3fdc3afdee18aec934021cfca117bcbd4dc2b6d3 Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Mon, 21 Jul 2025 08:42:02 -0500 Subject: [PATCH 13/25] spock module command fixes (#354) * resolve error with spock set-readonly * remove spock node-alter-location * add missing dependency --- cli/scripts/db.py | 1 + cli/scripts/spock.py | 33 +-------------------- docs/cli_functions.md | 1 - docs/functions/spock-node-alter-location.md | 11 ------- docs/functions/spock.md | 1 - 5 files changed, 2 insertions(+), 45 deletions(-) delete mode 100644 docs/functions/spock-node-alter-location.md diff --git a/cli/scripts/db.py b/cli/scripts/db.py index ae1e57d2..9f9bf0f5 100755 --- a/cli/scripts/db.py +++ b/cli/scripts/db.py @@ -9,6 +9,7 @@ import util import fire import re +import psycopg def create(db=None, User=None, Passwd=None, pg=None, spock=None, help=False): """ diff --git a/cli/scripts/spock.py b/cli/scripts/spock.py index 3376b66c..3bec2756 100644 --- a/cli/scripts/spock.py +++ b/cli/scripts/spock.py @@ -198,36 +198,6 @@ def node_drop(node_name, db): sys.exit(0) -def node_alter_location(node_name, location, db): - """Set location details for spock node.""" - pg_v,spock_v = get_spock_ver() - - [location_nm, country, state, lattitude, longitude] = util.get_location_dtls( - location - ) - - sql = """ -UPDATE spock.node - SET location_nm = ?, country = ?, state = ?, lattitude = ?, longitude = ? - WHERE location = ? -""" - - con = util.get_pg_connection(pg_v, db, util.get_user()) - - rc = 0 - try: - con = util.get_pg_connection(pg_v, "postgres", util.get_user()) - cur = con.cursor(row_factory=psycopg.rows.dict_row) - cur.execute(sql, [location_nm, country, state, lattitude, longitude]) - con.commit() - except Exception as e: - util.print_exception(e) - con.rollback() - rc = 1 - - sys.exit(rc) - - def node_list(db): """Display node table. @@ -726,7 +696,7 @@ def set_readonly(readonly="off"): util.message("spock.set_readonly() deprecated, use db.set_readonly() instead", "warning") - return(db.set_readonly(readonly, pg)) + return(db.set_readonly(readonly)) def get_pii_cols(db, schema=None): @@ -1017,7 +987,6 @@ def metrics_check(db): { "node-create": node_create, "node-drop": node_drop, - "node-alter-location": node_alter_location, "node-list": node_list, "node-add-interface": node_add_interface, "node-drop-interface": node_drop_interface, diff --git a/docs/cli_functions.md b/docs/cli_functions.md index 99162704..f7f5abdc 100644 --- a/docs/cli_functions.md +++ b/docs/cli_functions.md @@ -96,7 +96,6 @@ Use commands in this section to invoke spock extension functionality with the CL |---------|-------------| | [spock node-create](functions/spock-node-create.md) | Define a node for spock. | | [spock node-drop](functions/spock-node-drop.md) | Remove a spock node. | -| [spock node-alter-location](functions/spock-node-alter-location.md) | Set location details for spock node. | | [spock node-list](functions/spock-node-list.md) | Display node table. | | [spock node-add-interface](functions/spock-node-add-interface.md) | Add a new node interface. | | [spock node-drop-interface](functions/spock-node-drop-interface.md) | Delete a node interface. | diff --git a/docs/functions/spock-node-alter-location.md b/docs/functions/spock-node-alter-location.md deleted file mode 100644 index 9d39297b..00000000 --- a/docs/functions/spock-node-alter-location.md +++ /dev/null @@ -1,11 +0,0 @@ - -## SYNOPSIS - ./pgedge spock node-alter-location NODE_NAME LOCATION DB - -## DESCRIPTION - Set location details for spock node. - -## POSITIONAL ARGUMENTS - NODE_NAME - LOCATION - DB diff --git a/docs/functions/spock.md b/docs/functions/spock.md index d1f26fdc..5ed3198c 100644 --- a/docs/functions/spock.md +++ b/docs/functions/spock.md @@ -6,7 +6,6 @@ COMMAND is one of the following: node-create # Define a node for spock. node-drop # Remove a spock node. - node-alter-location # Set location details for spock node. node-list # Display node table. node-add-interface # Add a new node interface. node-drop-interface # Delete a node interface. From 4829eec76357f0d253c1e7ac4e899b82e3959d1c Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Thu, 24 Jul 2025 01:24:48 +0500 Subject: [PATCH 14/25] add validation to um install for spock5 upgrade (#356) * major spock version upgrade command (draft) * successfully upgraded to spock5 using um upgrade-spock * spock 5 upgrade support * Update um.py Upgrade-spock function hide * code refactored * additional test case added for spock 5 upgrade comp check * additional test case added for spock 5 upgrade comp check * additional test case added for spock 5 upgrade comp check * warning banner removed from case 3 * final adjustment to logging output --------- Co-authored-by: Matthew Mols --- cli/scripts/um.py | 90 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/cli/scripts/um.py b/cli/scripts/um.py index f07aedbc..ceff8e3a 100755 --- a/cli/scripts/um.py +++ b/cli/scripts/um.py @@ -3,8 +3,9 @@ import os, sys, glob, sqlite3, time import fire, meta, util - isJSON = util.isJSON +import re +import sqlite3 as _sqlite3 MY_HOME = util.MY_HOME @@ -83,20 +84,29 @@ def update(): run_cmd("update") + +# Match both "spock50" and "spock50-pg17" (and variants like spock5, spock5-pg16) +_SPOCK5_NAME_RE = re.compile(r"^spock5(?:0)?(?:-pg\d+)?$", re.IGNORECASE) +_SPOCK5_VER_RE = re.compile(r"^5\.", re.IGNORECASE) +_SPOCK50_RE = re.compile(r"^spock50(?:-pg\d+)?$", re.IGNORECASE) + def install(component, active=True): """Install a component.""" + # Trigger pre-check ONLY for Spock 5.0 artifacts + if _SPOCK50_RE.match(component): + validate_spock_upgrade() # should sys.exit(1) on failure; otherwise just returns + + # Common path (no duplication) if active not in (True, False): util.exit_message("'active' parm must be True or False") - - cmd = "install" - if active is False: - cmd = "install --no-preload" + cmd = "install" if active else "install --no-preload" util.message(f"um.install({cmd} {component})", "debug") run_cmd(cmd, component) + def remove(component): """Uninstall a component.""" installed_comp_list = meta.get_component_list() @@ -234,6 +244,76 @@ def verify_metadata(Project="", Stage="prod", IsCurrent=0): meta.pretty_sql(sql) +def validate_spock_upgrade(): + """ + Validate Spock↔PostgreSQL compatibility for an upcoming Spock 5 install. + """ + + DB_PATH = "data/conf/db_local.db" + + SPOCK_SQL = """ + SELECT version AS spock_ver, component AS spock_comp + FROM components + WHERE component LIKE 'spock%' + ORDER BY CASE + WHEN version LIKE '5.%' THEN 5 + WHEN version LIKE '4.%' THEN 4 + ELSE 0 + END DESC, + version DESC + LIMIT 1; + """ + + PG_SQL = """ + SELECT version AS pg_ver + FROM components + WHERE component LIKE 'pg__' + ORDER BY CAST(substr(component, 3) AS INTEGER) DESC + LIMIT 1; + """ + + try: + with _sqlite3.connect(DB_PATH) as conn: + sp_row = conn.execute(SPOCK_SQL).fetchone() + pg_row = conn.execute(PG_SQL).fetchone() + except _sqlite3.Error as err: + sys.exit(f"ERROR: SQLite query failed: {err}") + + if not pg_row: + sys.exit("ERROR: No PostgreSQL version row found.") + + pg_ver = pg_row[0] + spock_ver, spock_comp = (sp_row or (None, None)) + + # Already on Spock 5? No-op. + if spock_ver and ( + _SPOCK5_NAME_RE.match(spock_comp) or _SPOCK5_VER_RE.match(spock_ver) + ): + return 0 + + # Downtime warning + banner = "=" * 80 + + if not spock_ver: + # Case 1: no existing Spock installed, run validation but don't print banner + try: + util.validate_spock_pg_compat('50', pg_ver) + except Exception as exc: + sys.exit(f"ERROR: Compatibility check failed: {exc}") + + elif spock_ver.startswith('4'): + # Case 2: Spock 4.x installed, this is a major upgrade + print(f"\n{banner}") + print("*** WARNING: This operation will cause downtime! ***") + print(f"{banner}\n") + print(f"Detected Spock version {spock_ver} on PostgreSQL {pg_ver}") + try: + util.validate_spock_pg_compat(spock_ver, pg_ver) + except Exception as exc: + sys.exit(f"ERROR: Compatibility check failed: {exc}") + + print("Compatibility check passed.") + return 0 if __name__ == "__main__": fire.Fire( From 84f845705920f8ca1744dcf3c86e7f5c94dc5b2c Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Fri, 25 Jul 2025 07:19:25 -0500 Subject: [PATCH 15/25] update python deps minor versions (#355) --- requirements.txt | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/requirements.txt b/requirements.txt index 604ab605..8f9a6261 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,41 +1,41 @@ ## Core CLI ######### -requests==2.32.2 -paramiko==3.5.0 -psycopg==3.2.3; platform_system=="Linux" -psycopg-binary==3.2.3; platform_system=="Linux" -psutil==6.1.0 +requests==2.32.4 +paramiko==3.5.1 +psycopg==3.2.9; platform_system=="Linux" +psycopg-binary==3.2.9; platform_system=="Linux" +psutil==6.1.1 pypsutil==0.2.0 semantic_version==2.10.0 -six==1.16.0 +six==1.17.0 termcolor==2.5.0 -typing_extensions==4.12.2 -click==8.1.7 +typing_extensions==4.14.1 +click==8.2.1 tabulate==0.9.0 -python-crontab==3.2.0 -certifi==2024.7.4 -charset_normalizer==3.3.2 +python-crontab==3.3.0 +certifi==2024.12.14 +charset_normalizer==3.4.2 gpustat==1.1.1 py-cpuinfo==9.0.0 -prettytable==3.12.0 -Flask==3.0.3 -minio==7.2.10 -tqdm==4.67.0 +prettytable==3.16.0 +Flask==3.1.1 +minio==7.2.15 +tqdm==4.67.1 ## ACE Advanced ##### ordered-set==4.1.0 mpire[dashboard]==2.10.2 rich==13.9.4 -apscheduler==3.10.4 -pandas==2.2.3 +apscheduler==3.11.0 +pandas==2.3.1 pyopenssl==24.3.0 -packaging==23.1 +packaging==23.2 ## pgEdge-HA ######## cdiff==1.0 -urllib3==2.2.2 -PyYAML==6.0.1 +urllib3==2.5.0 +PyYAML==6.0.2 python-etcd==0.4.5 python-dateutil==2.9.0 -cryptography==44.0.1 -ydiff==1.3 +cryptography==44.0.3 +ydiff==1.4.2 From b06c31928aaadd164a2182197b730bead3dc398d Mon Sep 17 00:00:00 2001 From: hayee-bhatti Date: Wed, 30 Jul 2025 16:29:07 +0500 Subject: [PATCH 16/25] PLAT-184 remove an unused dependency --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8f9a6261..0c521bfc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,6 @@ semantic_version==2.10.0 six==1.17.0 termcolor==2.5.0 typing_extensions==4.14.1 -click==8.2.1 tabulate==0.9.0 python-crontab==3.3.0 certifi==2024.12.14 From d5bb2ae850a83137816b7a3472038d9d8649cf51 Mon Sep 17 00:00:00 2001 From: hayee-bhatti Date: Wed, 30 Jul 2025 16:31:57 +0500 Subject: [PATCH 17/25] PLAT-192 Bump ctlibs version to 1.7 --- env.sh | 2 +- src/conf/versions.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/env.sh b/env.sh index 07a0eb0a..c7ee9792 100755 --- a/env.sh +++ b/env.sh @@ -6,7 +6,7 @@ kirkV=$hubV bundle=pgedge api=pgedge -ctlibsV=1.6 +ctlibsV=1.7 spock50V=5.0.0-1 diff --git a/src/conf/versions.sql b/src/conf/versions.sql index f10e9127..6c31e133 100644 --- a/src/conf/versions.sql +++ b/src/conf/versions.sql @@ -405,7 +405,7 @@ INSERT INTO versions VALUES ('hypopg-pg16', '1.4.1-1', 'amd, arm', 1, '20230509 INSERT INTO projects VALUES ('ctlibs', 'pge', 0, 0, '', 3, 'https://github.com/pgedge/cli', 'ctlibs', 0, 'ctlibs.png', 'ctlibs', 'https://github.com/pgedge/cli', ''); INSERT INTO releases VALUES ('ctlibs', 2, 'ctlibs', 'pgEdge Libs', '', 'prod', '', 1, '', '', ''); -INSERT INTO versions VALUES ('ctlibs', '1.6', '', 1, '20240925', '', '', ''); +INSERT INTO versions VALUES ('ctlibs', '1.7', '', 1, '20250729', '', '', ''); -- ## PGCAT ############################# INSERT INTO projects VALUES ('pgcat', 'pge', 11, 5433, '', 3, 'https://github.com/pgedge/pgcat/tags', From 39daccee0a456e7db8ecf8f39ab28b1f1092c15d Mon Sep 17 00:00:00 2001 From: Tej Kashi Date: Fri, 1 Aug 2025 13:27:11 -0400 Subject: [PATCH 18/25] Merge pull request #360 from pgEdge/ace/datatype-fix * Use conservative datatype handling * Add configurable option for using repeatable read while updating Merkle trees --- cli/scripts/ace-tests/test_data_types.py | 632 +++++++++++++++++++---- cli/scripts/ace.py | 129 +++-- cli/scripts/ace_config.py | 3 + cli/scripts/ace_core.py | 8 +- cli/scripts/ace_mtree.py | 9 +- 5 files changed, 594 insertions(+), 187 deletions(-) diff --git a/cli/scripts/ace-tests/test_data_types.py b/cli/scripts/ace-tests/test_data_types.py index 950bf2dd..ff5d7aec 100644 --- a/cli/scripts/ace-tests/test_data_types.py +++ b/cli/scripts/ace-tests/test_data_types.py @@ -1,3 +1,6 @@ +from datetime import datetime, timedelta, date, time +from decimal import Decimal +from ipaddress import IPv4Address import pytest import psycopg import json @@ -31,50 +34,25 @@ def setup_datatypes(self, nodes): bytea_col BYTEA, point_col POINT, text_col TEXT, - text_array_col TEXT[] + text_array_col TEXT[], + bool_col BOOLEAN, + bigint_col BIGINT, + smallint_col SMALLINT, + numeric_col NUMERIC(10, 4), + real_col REAL, + time_col TIME, + date_col DATE, + timestamp_col TIMESTAMP, + interval_col INTERVAL, + inet_col INET, + macaddr_col MACADDR, + money_col MONEY ) """ ) # Insert sample data - cur.execute( - """ - INSERT INTO datatypes_test VALUES - ( - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - 42, - 3.14159, - ARRAY[1, 2, 3, 4, 5], - '{"key": "value", "nested": {"foo": "bar"}}', - decode('DEADBEEF', 'hex'), - point(1.5, 2.5), - 'sample text', - ARRAY['apple', 'banana', 'cherry'] - ), - ( - 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12', - 100, - 2.71828, - ARRAY[10, 20, 30], - '{"numbers": [1, 2, 3], "active": true}', - decode('BADDCAFE', 'hex'), - point(3.7, 4.2), - 'another sample', - ARRAY['dog', 'cat', 'bird'] - ), - ( - 'c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a13', - -17, - 0.577216, - ARRAY[]::INTEGER[], - '{"empty": true}', - NULL, - point(0, 0), - 'third sample', - ARRAY[]::TEXT[] - ) - """ - ) + self._insert_initial_data(cur) repset_add_datatypes_sql = """ SELECT spock.repset_add_table('test_repset', 'datatypes_test') @@ -106,6 +84,145 @@ def setup_datatypes(self, nodes): except Exception as e: pytest.fail(f"Failed to setup/cleanup datatypes test: {str(e)}") + def _insert_initial_data(self, cur): + """Helper method to insert the initial dataset.""" + cur.execute(self._get_initial_data_sql()) + + def _get_initial_data_sql(self): + """Returns the SQL for inserting the initial dataset.""" + return """ + INSERT INTO datatypes_test VALUES + ( + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + 42, + 3.14159, + ARRAY[1, 2, 3, 4, 5], + '{"key": "value", "nested": {"foo": "bar"}}', + decode('DEADBEEF', 'hex'), + point(1.5, 2.5), + 'sample text', + ARRAY['apple', 'banana', 'cherry'], + true, + 9223372036854775807, + 32767, + 12345.6789, + 123.456, + '12:34:56', + '2024-01-01', + '2024-01-01 12:34:56', + '30 days', + '192.168.1.1', + '08:00:2b:01:02:03', + 12345.67 + ), + ( + 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12', + 100, + 2.71828, + ARRAY[10, 20, 30], + '{"numbers": [1, 2, 3], "active": true}', + decode('BADDCAFE', 'hex'), + point(3.7, 4.2), + 'another sample', + ARRAY['dog', 'cat', 'bird'], + false, + -9223372036854775808, + -32768, + -12345.6789, + -123.456, + '23:59:59', + '2023-12-31', + '2023-12-31 23:59:59', + '-5 days', + '10.0.0.1', + '00:1A:2B:3C:4D:5E', + -12345.67 + ), + ( + 'c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a13', + -17, + 0.577216, + ARRAY[]::INTEGER[], + '{"empty": true}', + NULL, + point(0, 0), + 'third sample', + ARRAY[]::TEXT[], + true, + 0, + 0, + 0, + 0, + '00:00:00', + '1970-01-01', + '1970-01-01 00:00:00', + '0 seconds', + '0.0.0.0', + '00:00:00:00:00:00', + 0 + ), + ( + 'd0eebc99-9c0b-4ef8-bb6d-6bb9bd380a14', + NULL, + NULL, + ARRAY[NULL, 1, NULL], + NULL, + NULL, + NULL, + 'null', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ), + ( + 'e0eebc99-9c0b-4ef8-bb6d-6bb9bd380a15', + 1, + 'NaN', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ) + """ + + def reset_data(self, nodes): + """Reset data in the test table before each test function.""" + try: + for node in nodes: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("TRUNCATE TABLE datatypes_test") + self._insert_initial_data(cur) + conn.commit() + cur.close() + conn.close() + except Exception as e: + pytest.fail(f"Failed to reset data between tests: {str(e)}") + # Override the table_name parameter for all parameterized tests @pytest.mark.parametrize("table_name", ["public.datatypes_test"]) def test_simple_table_diff(self, cli, capsys, table_name): @@ -113,16 +230,28 @@ def test_simple_table_diff(self, cli, capsys, table_name): @pytest.mark.parametrize("table_name", ["public.datatypes_test"]) @pytest.mark.parametrize( - "column_name,test_value", + "column_name,test_value,expected_diffs", [ - ("int_col", "9999"), - ("float_col", "123.456"), - ("array_col", "ARRAY[99, 98, 97]"), - ("json_col", '\'{"test": "modified"}\''), - ("bytea_col", "decode('FEEDFACE', 'hex')"), - ("point_col", "point(99.9, 99.9)"), - ("text_col", "'modified-text'"), - ("text_array_col", "ARRAY['modified', 'text', 'array']"), + ("int_col", "9999", 5), + ("float_col", "123.456", 5), + ("array_col", "ARRAY[99, 98, 97]", 5), + ("json_col", '\'{"test": "modified"}\'', 5), + ("bytea_col", "decode('FEEDFACE', 'hex')", 5), + ("point_col", "point(99.9, 99.9)", 5), + ("text_col", "'modified-text'", 5), + ("text_array_col", "ARRAY['modified', 'text', 'array']", 5), + ("bool_col", "false", 5), + ("bigint_col", "1234567890123456789", 5), + ("smallint_col", "-32768", 5), + ("numeric_col", "98765.4321", 5), + ("real_col", "987.654", 5), + ("time_col", "'11:22:33'", 5), + ("date_col", "'2025-05-25'", 5), + ("timestamp_col", "'2025-05-25 11:22:33'", 5), + ("interval_col", "'90 days'", 5), + ("inet_col", "'192.168.100.200'", 5), + ("macaddr_col", "'01:23:45:67:89:ab'", 5), + ("money_col", "9876.54", 5), ], ) @pytest.mark.parametrize("key_column", ["id"]) @@ -134,6 +263,7 @@ def test_table_diff_with_differences( table_name, column_name, test_value, + expected_diffs, key_column, diff_file_path, ): @@ -154,11 +284,6 @@ def test_table_diff_with_differences( """ ) - modified_rows = cur.fetchall() - modified_indices = { - str(row[0]) for row in modified_rows - } # Convert UUID to string - conn.commit() cur.close() conn.close() @@ -185,55 +310,79 @@ def test_table_diff_with_differences( # Verify number of differences assert ( - len(diff_data["diffs"]["n1/n2"]["n2"]) == 3 - ), "Expected 3 differences," + len(diff_data["diffs"]["n1/n2"]["n2"]) == expected_diffs + ), f"Expected {expected_diffs} differences," f" found {len(diff_data['diffs']['n1/n2']['n2'])}" - # Verify modified rows are in diff - diff_indices = { - str(diff["id"]) for diff in diff_data["diffs"]["n1/n2"]["n2"] - } - assert ( - modified_indices == diff_indices - ), "Modified rows don't match diff file records" - # Verify the differences are correctly reported for diff in diff_data["diffs"]["n1/n2"]["n2"]: + diff_val = diff[column_name] + expected_val_str = test_value.strip("'") + if column_name == "json_col": assert ( - diff[column_name].get("test") == "modified" + diff_val.get("test") == "modified" ), f"Modified row {diff['id']} doesn't have expected JSON value" elif column_name == "array_col": - assert diff[column_name] == [ + assert diff_val == [ 99, 98, 97, ], f"Modified row {diff['id']} doesn't have expected array value" elif column_name == "text_array_col": - assert diff[column_name] == [ + assert diff_val == [ "modified", "text", "array", ], ( - f"Modified row {diff['id']} doesn't have expected " - "text array value" + f"Modified row {diff['id']} doesn't have expected text" + " array value" ) elif column_name == "point_col": assert ( - diff[column_name] == "(99.9,99.9)" + diff_val == "(99.9,99.9)" ), f"Modified row {diff['id']} doesn't have expected point value" elif column_name == "bytea_col": - print("bytea col: ", diff[column_name]) assert ( - diff[column_name] == "feedface" + diff_val == "feedface" ), f"Modified row {diff['id']} doesn't have expected bytea value" + elif column_name == "macaddr_col": + assert ( + diff_val == "01:23:45:67:89:ab" + ), f"Modified row {diff['id']} doesn't have expected macaddr value" + elif column_name == "money_col": + cleaned_diff = diff_val.replace("$", "").replace(",", "") + assert float(cleaned_diff) == float( + expected_val_str + ), f"Modified row {diff['id']} doesn't have expected money value" + elif column_name == "bool_col": + assert str(diff_val).lower() == expected_val_str.lower(), ( + f"Modified row {diff['id']} " + "doesn't have expected boolean value" + ) + elif column_name == "interval_col": + # Interval representation can vary, so we check equality + # directly in Postgres + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + "SELECT %s::interval = %s::interval", + (str(diff_val), expected_val_str), + ) + is_equal = cur.fetchone()[0] + cur.close() + conn.close() + assert is_equal, ( + f"Modified row {diff['id']} " + f"doesn't have expected interval value. " + f"Got {diff_val}, expected equivalence to {expected_val_str}" + ) else: - assert str(diff[column_name]) in ( - test_value.strip("'"), - "9999", - "123.456", - "modified-text", - ), f"Modified row {diff['id']} doesn't have expected value" + assert str(diff_val) == expected_val_str, ( + f"Modified row {diff['id']} " + f"doesn't have expected value, got {diff_val} " + f"expected {expected_val_str}" + ) except Exception as e: pytest.fail(f"Failed to test differences for {column_name}: {str(e)}") @@ -245,16 +394,28 @@ def test_simple_table_repair(self, cli, capsys, table_name, diff_file_path): @pytest.mark.parametrize("table_name", ["public.datatypes_test"]) @pytest.mark.parametrize("key_column", ["id"]) @pytest.mark.parametrize( - "column_name,test_value", + "column_name,test_value,expected_rerun_diffs", [ - ("int_col", "1234"), - ("float_col", "98.765"), - ("array_col", "ARRAY[11, 22, 33]"), - ("json_col", '\'{"rerun": "modified"}\''), - ("bytea_col", "decode('ABCDEF12', 'hex')"), - ("point_col", "point(88.8, 88.8)"), - ("text_col", "'rerun-modified'"), - ("text_array_col", "ARRAY['rerun', 'modified', 'array']"), + ("int_col", "1234", 5), + ("float_col", "98.765", 5), + ("array_col", "ARRAY[11, 22, 33]", 5), + ("json_col", '\'{"rerun": "modified"}\'', 5), + ("bytea_col", "decode('ABCDEF12', 'hex')", 5), + ("point_col", "point(88.8, 88.8)", 5), + ("text_col", "'rerun-modified'", 5), + ("text_array_col", "ARRAY['rerun', 'modified', 'array']", 5), + ("bool_col", "true", 5), + ("bigint_col", "-1234567890123456789", 5), + ("smallint_col", "32767", 5), + ("numeric_col", "-98765.4321", 5), + ("real_col", "-987.654", 5), + ("time_col", "'01:02:03'", 5), + ("date_col", "'2022-02-02'", 5), + ("timestamp_col", "'2022-02-02 01:02:03'", 5), + ("interval_col", "'60 days'", 5), + ("inet_col", "'127.0.0.1'", 5), + ("macaddr_col", "'fe:dc:ba:98:76:54'", 5), + ("money_col", "-9876.54", 5), ], ) def test_table_rerun_temptable( @@ -266,6 +427,7 @@ def test_table_rerun_temptable( key_column, column_name, test_value, + expected_rerun_diffs, diff_file_path, ): """Test table rerun temptable with various data types""" @@ -323,42 +485,115 @@ def test_table_rerun_temptable( diff_data = json.load(f) assert ( - len(diff_data["diffs"]["n1/n2"]["n2"]) == 3 - ), f"Expected 3 differences, found {len(diff_data['diffs']['n1/n2']['n2'])}" + len(diff_data["diffs"]["n1/n2"]["n2"]) == expected_rerun_diffs + ), f"Expected {expected_rerun_diffs} differences, " + f"found {len(diff_data['diffs']['n1/n2']['n2'])}" # Verify the differences are correctly reported for diff in diff_data["diffs"]["n1/n2"]["n2"]: + diff_val = diff[column_name] + expected_val_str = test_value.strip("'") + if column_name == "json_col": assert ( - diff[column_name].get("rerun") == "modified" + diff_val.get("rerun") == "modified" ), f"Modified row {diff['id']} doesn't have expected JSON value" elif column_name == "array_col": - assert diff[column_name] == [ + assert diff_val == [ 11, 22, 33, ], f"Modified row {diff['id']} doesn't have expected array value" elif column_name == "text_array_col": - assert diff[column_name] == [ + assert diff_val == [ "rerun", "modified", "array", ], f"Modified row {diff['id']} doesn't have expected text array value" elif column_name == "point_col": assert ( - diff[column_name] == "(88.8,88.8)" + diff_val == "(88.8,88.8)" ), f"Modified row {diff['id']} doesn't have expected point value" elif column_name == "bytea_col": assert ( - diff[column_name] == "abcdef12" + diff_val == "abcdef12" ), f"Modified row {diff['id']} doesn't have expected bytea value" + elif column_name == "macaddr_col": + assert ( + diff_val == "fe:dc:ba:98:76:54" + ), f"Modified row {diff['id']} doesn't have expected macaddr value" + elif column_name == "money_col": + cleaned_diff = diff_val.replace("$", "").replace(",", "") + assert float(cleaned_diff) == float( + expected_val_str + ), f"Modified row {diff['id']} doesn't have expected money value" + elif column_name == "bool_col": + assert ( + str(diff_val).lower() == expected_val_str.lower() + ), f"Modified row {diff['id']} doesn't have expected boolean value" + elif column_name == "interval_col": + # Interval representation can vary, so we check equality in the DB + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + "SELECT %s::interval = %s::interval", + (str(diff_val), expected_val_str), + ) + is_equal = cur.fetchone()[0] + cur.close() + conn.close() + assert is_equal, ( + f"Modified row {diff['id']} " + f"doesn't have expected interval value. " + f"Got {diff_val}, expected equivalence to {expected_val_str}" + ) else: - assert str(diff[column_name]) in ( - test_value.strip("'"), - "1234", - "98.765", - "rerun-modified", - ), f"Modified row {diff['id']} doesn't have expected value" + assert str(diff_val) == expected_val_str, ( + f"Modified row {diff['id']} " + f"doesn't have expected value, got {diff_val} " + f"expected {expected_val_str}" + ) + + def _verify_repaired_value(self, column_name, repaired_value, expected_value): + """Helper function to verify repaired values based on data type""" + if column_name == "bytea_col": + assert ( + repaired_value == expected_value + ), "Repaired bytea value doesn't match expected value" + elif column_name == "point_col": + if isinstance(repaired_value, str): + repaired_tuple = tuple( + map(float, repaired_value.strip("()").split(",")) + ) + else: + repaired_tuple = repaired_value + + expected_tuple = tuple(map(float, expected_value.strip("()").split(","))) + assert ( + repaired_tuple == expected_tuple + ), "Repaired point value doesn't match expected value" + elif column_name == "numeric_col": + assert repaired_value == Decimal( + expected_value + ), f"Repaired value for {column_name} doesn't match" + elif column_name == "inet_col": + assert repaired_value == IPv4Address( + expected_value + ), f"Repaired value for {column_name} doesn't match" + elif column_name in ["time_col", "date_col", "timestamp_col", "interval_col"]: + assert ( + repaired_value == expected_value + ), f"Repaired value for {column_name} doesn't match" + elif column_name == "money_col": + cleaned_repaired = str(repaired_value).replace("$", "").replace(",", "") + cleaned_expected = str(expected_value).replace("$", "").replace(",", "") + assert float(cleaned_repaired) == float( + cleaned_expected + ), f"Repaired money value doesn't match for {column_name}" + else: + assert ( + repaired_value == expected_value + ), f"Repaired value doesn't match expected value for {column_name}" @pytest.mark.parametrize("table_name", ["public.datatypes_test"]) @pytest.mark.parametrize( @@ -376,6 +611,22 @@ def test_table_rerun_temptable( "ARRAY['modified', 'text', 'array']", ["modified", "text", "array"], ), + ("bool_col", "false", False), + ("bigint_col", "1234567890123456789", 1234567890123456789), + ("smallint_col", "-32768", -32768), + ("numeric_col", "98765.4321", Decimal("98765.4321")), + ("real_col", "987.654", 987.654), + ("time_col", "'11:22:33'", time(11, 22, 33)), + ("date_col", "'2025-05-25'", date(2025, 5, 25)), + ( + "timestamp_col", + "'2025-05-25 11:22:33'", + datetime(2025, 5, 25, 11, 22, 33), + ), + ("interval_col", "'90 days'", timedelta(days=90)), + ("inet_col", "'192.168.100.200'", "192.168.100.200"), + ("macaddr_col", "'01:23:45:67:89:ab'", "01:23:45:67:89:ab"), + ("money_col", "9876.54", "$9,876.54"), ], ) def test_table_repair_datatypes( @@ -441,18 +692,177 @@ def test_table_repair_datatypes( conn.close() # Compare with expected value - if column_name == "bytea_col": - assert ( - repaired_value == expected_value - ), "Repaired bytea value doesn't match expected value" - elif column_name == "point_col": - assert ( - str(repaired_value) == expected_value - ), "Repaired point value doesn't match expected value" - else: - assert ( - repaired_value == expected_value - ), f"Repaired value doesn't match expected value for {column_name}" + self._verify_repaired_value(column_name, repaired_value, expected_value) except Exception as e: pytest.fail(f"Test failed: {str(e)}") + + @pytest.mark.parametrize("id_to_update", ["d0eebc99-9c0b-4ef8-bb6d-6bb9bd380a14"]) + def test_null_and_string_literal_handling( + self, cli, capsys, diff_file_path, id_to_update + ): + """ + Verify that NULL values and string literals like 'null' are + handled correctly. + """ + try: + # Our prior repair unfortunately reset a lot of fields, so we reset + # the data first here + self.reset_data(nodes=["n1", "n2"]) + + # On n2, update text_col from 'null' to 'not null' and + # int_col from NULL to a number + conn = psycopg.connect(host="n2", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + # This specific id has "null" as a literal in the text_col + cur.execute( + """ + UPDATE datatypes_test + SET text_col = 'not null anymore', int_col = 123 + WHERE id = %s + """, + (id_to_update,), + ) + conn.commit() + cur.close() + conn.close() + + # Run table-diff + cli.table_diff(cluster_name="eqn-t9da", table_name="public.datatypes_test") + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file_path.path = match.group(1) + + # Verify diff file + with open(diff_file_path.path, "r") as f: + diff_data = json.load(f) + + diffs_n1 = diff_data["diffs"]["n1/n2"]["n1"] + diffs_n2 = diff_data["diffs"]["n1/n2"]["n2"] + + assert len(diffs_n1) == 1, "Expected 1 difference on n1" + assert len(diffs_n2) == 1, "Expected 1 difference on n2" + + # Check n1 (original values) + assert diffs_n1[0]["id"] == id_to_update + assert diffs_n1[0]["text_col"] == "null" + assert diffs_n1[0]["int_col"] is None + + # Check n2 (modified values) + assert diffs_n2[0]["id"] == id_to_update + assert diffs_n2[0]["text_col"] == "not null anymore" + assert diffs_n2[0]["int_col"] == 123 + + # Run table-repair + cli.table_repair( + cluster_name="eqn-t9da", + table_name="public.datatypes_test", + diff_file=diff_file_path.path, + source_of_truth="n2", + ) + + # Verify repair on n1 + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + """ + SELECT text_col, int_col FROM datatypes_test + WHERE id = %s + """, + (id_to_update,), + ) + repaired_text, repaired_int = cur.fetchone() + cur.close() + conn.close() + + assert repaired_text == "not null anymore" + assert repaired_int == 123 + + except Exception as e: + pytest.fail(f"Test for null handling failed: {str(e)}") + + @pytest.mark.parametrize("id_to_update", ["e0eebc99-9c0b-4ef8-bb6d-6bb9bd380a15"]) + def test_ast_literal_eval_fallback(self, cli, capsys, diff_file_path, id_to_update): + """ + Verify that the fallback to string representation works when + ast.literal_eval fails. + """ + try: + # Resetting again here + self.reset_data(nodes=["n1", "n2"]) + + # On n2, update float_col from NaN to a valid number + conn = psycopg.connect(host="n2", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + cur.execute( + """ + UPDATE datatypes_test + SET float_col = 1.23 + WHERE id = %s + """, + (id_to_update,), + ) + conn.commit() + cur.close() + conn.close() + + # Run table-diff + cli.table_diff(cluster_name="eqn-t9da", table_name="public.datatypes_test") + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file_path.path = match.group(1) + + # Verify diff file + with open(diff_file_path.path, "r") as f: + diff_data = json.load(f) + + diffs_n1 = diff_data["diffs"]["n1/n2"]["n1"] + diffs_n2 = diff_data["diffs"]["n1/n2"]["n2"] + + assert len(diffs_n1) == 1, "Expected 1 difference on n1" + assert len(diffs_n2) == 1, "Expected 1 difference on n2" + + # Check n1 (original 'NaN' value) + assert diffs_n1[0]["id"] == id_to_update + assert diffs_n1[0]["float_col"] == "nan" + + # Check n2 (modified value) + assert diffs_n2[0]["id"] == id_to_update + assert diffs_n2[0]["float_col"] == 1.23 + + # Run table-repair + cli.table_repair( + cluster_name="eqn-t9da", + table_name="public.datatypes_test", + diff_file=diff_file_path.path, + source_of_truth="n2", + ) + + # Verify repair on n1 + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + """ + SELECT float_col FROM datatypes_test + WHERE id = %s + """, + (id_to_update,), + ) + repaired_float = cur.fetchone()[0] + cur.close() + conn.close() + + assert repaired_float == 1.23 + + except Exception as e: + pytest.fail(f"Test for ast.literal_eval fallback failed: {str(e)}") diff --git a/cli/scripts/ace.py b/cli/scripts/ace.py index ecf50d43..0fed3a05 100755 --- a/cli/scripts/ace.py +++ b/cli/scripts/ace.py @@ -727,63 +727,47 @@ def check_diff_file_format(diff_file_path: str, task) -> dict: return diff_json -def convert_pg_type_to_json(item: str, type: str): +def convert_pg_type_to_json(item, type): """ Converts a value from a postgres column to a json-compatible type. """ # TODO: Need to revisit this. - try: - # List of types that should be treated as strings - string_types = [ - "char", - "text", - "time", - "bytea", - "uuid", - "date", - "timestamp", - "interval", - "inet", - "macaddr", - "xml", - "money", - "point", - "line", - "polygon", - ] - - # Types that can be directly represented in JSON - json_compatible_types = [ - "json", - "jsonb", - "boolean", - "integer", - "bigint", - "smallint", - "numeric", - "real", - "double precision", - ] - - type_lower = type.lower() - - if not item or item == "" or item.lower() == "null" or item.lower() == "none": - return None - elif "[]" in type_lower: - return ast.literal_eval(item) - elif any(s in type_lower for s in json_compatible_types): - # For JSON-compatible types, parse them using AST - return ast.literal_eval(item) - elif any(s in type_lower for s in string_types): - return item - else: - # Default to treating as string if type is unknown - return item + type_lower = type.lower() - except Exception as e: - raise AceException( - f"Could not convert value {item} to {type} while writing to json: {e}" - ) + # Types that should be parsed into native JSON types (not strings) + json_compatible_types = [ + "json", + "jsonb", + "boolean", + "integer", + "bigint", + "smallint", + "numeric", + "real", + "double precision", + ] + + is_parsable = ( + any(s in type_lower for s in json_compatible_types) or "[]" in type_lower + ) + + if not is_parsable: + # For string-like types (text, varchar, etc.), we return the value + # directly. This correctly preserves string literals like 'None' or + # 'null'. A database NULL would arrive here as item=None from the driver. + return item + + # For parsable types (numeric, boolean, json, array), we can interpret + # 'null' and 'none'. + if item is None or str(item).lower() in ("", "null", "none"): + return None + + try: + # For JSON-compatible types, parse them using AST + return ast.literal_eval(str(item)) + except (ValueError, SyntaxError): + # If conversion fails, treat as a string + return str(item) def convert_json_to_pg_type(rows, cols_list, col_types) -> list[tuple]: @@ -841,31 +825,40 @@ def convert_json_to_pg_type(rows, cols_list, col_types) -> list[tuple]: modified_row = tuple() for col_name in cols_list: col_type = col_types[col_name] - elem = str(row[col_name]) + elem = row[col_name] + type_lower = col_type.lower() try: - type_lower = col_type.lower() - - if ( - not elem - or elem == "" - or elem.lower() == "null" - or elem.lower() == "none" - ): - modified_row += (None,) - elif "[]" in type_lower: - modified_row += (ast.literal_eval(elem),) - elif any(s in type_lower for s in string_types): + # If the column type is a string type, we don't need to do anything + # special. A value of None will be converted to NULL by psycopg. + if any(s in type_lower for s in string_types): if type_lower == "bytea": - modified_row += (bytes.fromhex(elem),) + # We stored bytea as hex, so we need to convert it back + if elem is not None: + modified_row += (bytes.fromhex(elem),) + else: + modified_row += (None,) else: modified_row += (elem,) + continue + + # For non-string types, if the value is None, or looks like null, + # it should be treated as such. + if elem is None or str(elem).lower() in ("null", "none", ""): + modified_row += (None,) + continue + + elem_str = str(elem) + + if "[]" in type_lower: + modified_row += (ast.literal_eval(elem_str),) elif any(s in type_lower for s in json_compatible_types): - item = ast.literal_eval(elem) - if type_lower == "jsonb" or type_lower == "json": + item = ast.literal_eval(elem_str) + if type_lower in ("jsonb", "json"): item = json.dumps(item) modified_row += (item,) else: + # Fallback for any other types modified_row += (elem,) except (ValueError, SyntaxError): diff --git a/cli/scripts/ace_config.py b/cli/scripts/ace_config.py index 1580b1c4..39379e0e 100644 --- a/cli/scripts/ace_config.py +++ b/cli/scripts/ace_config.py @@ -12,6 +12,9 @@ STATEMENT_TIMEOUT = 0 # in milliseconds CONNECTION_TIMEOUT = 10 # in seconds +# Whether to use repeatable read isolation for Merkle tree updates +USE_REPEATABLE_READ = False + # Default values for ACE table-diff MAX_DIFF_ROWS = 1_000_000 MIN_DIFF_BLOCK_SIZE = 1000 diff --git a/cli/scripts/ace_core.py b/cli/scripts/ace_core.py index 91567668..b0c35b45 100644 --- a/cli/scripts/ace_core.py +++ b/cli/scripts/ace_core.py @@ -574,12 +574,16 @@ def compare_checksums(worker_id, shared_objects, worker_state, pkey1, pkey2): for row_key in t1_only: worker_diffs[node_pair_key][host1].append( - dict(zip(cols, (str(x) for x in row_key))) + dict( + zip(cols, (str(x) if x is not None else None for x in row_key)) + ) ) for row_key in t2_only: worker_diffs[node_pair_key][host2].append( - dict(zip(cols, (str(x) for x in row_key))) + dict( + zip(cols, (str(x) if x is not None else None for x in row_key)) + ) ) total_diffs += max(len(t1_only), len(t2_only)) diff --git a/cli/scripts/ace_mtree.py b/cli/scripts/ace_mtree.py index a621b66b..32eb7980 100644 --- a/cli/scripts/ace_mtree.py +++ b/cli/scripts/ace_mtree.py @@ -1025,7 +1025,6 @@ def split_blocks(conn, schema, table, key, blocks, block_size): i += 1 - conn.commit() pbar.close() return list(modified_positions) @@ -1399,7 +1398,6 @@ def merge_blocks(conn, schema, table, key, blocks, block_size): if i >= len(blocks): break - conn.commit() pbar.close() return list(modified_positions) @@ -1408,7 +1406,7 @@ def update_mtree(mtree_task: MerkleTreeTask, skip_all_checks=False) -> None: """ Update a Merkle tree by recomputing hashes for dirty leaf nodes and new blocks. Also processes any pending block rebalancing operations. - Uses repeatable read isolation to ensure consistency during the update. + Uses repeatable read isolation if config.USE_REPEATABLE_READ is True. Args: cluster_name (str): Name of the cluster @@ -1455,7 +1453,8 @@ def update_mtree(mtree_task: MerkleTreeTask, skip_all_checks=False) -> None: for node in mtree_task.fields.cluster_nodes: _, conn = mtree_task.connection_pool.connect(node) - conn.set_isolation_level(IsolationLevel.REPEATABLE_READ) + if config.USE_REPEATABLE_READ: + conn.set_isolation_level(IsolationLevel.REPEATABLE_READ) print(f"\nUpdating Merkle tree on node: {node['name']}") @@ -1476,7 +1475,6 @@ def update_mtree(mtree_task: MerkleTreeTask, skip_all_checks=False) -> None: if not blocks_to_update: print(f"No updates needed for {node['name']}") - conn.commit() continue # First identify blocks that might need splitting based on insert count @@ -1557,7 +1555,6 @@ def update_mtree(mtree_task: MerkleTreeTask, skip_all_checks=False) -> None: if not blocks_to_update: print(f"No updates needed for {node['name']}") - conn.commit() continue print(f"Found {len(blocks_to_update)} blocks to update") From 5f9d01d39c4e0ca5b49bf329c8725462d0ee63f0 Mon Sep 17 00:00:00 2001 From: Hayee Bhatti <152845623+hayee-bhatti@users.noreply.github.com> Date: Fri, 1 Aug 2025 23:20:10 +0500 Subject: [PATCH 19/25] [BR-158]: Introduce spock60 builds in CLI (#362) Build script changes to produce spock60 builds from the main branch. Updated versioning (6.0.0-devel) and metadata. Updated 'current' builds workflow to now produce spock60 from main. Although, spock60 is hidden in the um list output, it can be installed in the current builds (only) by passing --spock_ver=6.0.0 in the pgedge setup command. --- .../workflows/current-amd8-daily-build-devel.yml | 8 ++++---- .../workflows/current-arm9-daily-build-devel.yml | 8 ++++---- build.sh | 3 +++ env.sh | 2 ++ src/conf/versions.sql | 14 ++++++++++++++ 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/current-amd8-daily-build-devel.yml b/.github/workflows/current-amd8-daily-build-devel.yml index 76a334ee..08b772be 100644 --- a/.github/workflows/current-amd8-daily-build-devel.yml +++ b/.github/workflows/current-amd8-daily-build-devel.yml @@ -4,7 +4,7 @@ name: Current Daily Build Devel - amd8 env: DEFAULT_CLI_BRANCH: "main" # Default CLI branch for scheduled runs DEFAULT_MODE: "current" # Always "current" for this workflow - DEFAULT_COMPONENT: "spock50" # Default spock component name + DEFAULT_COMPONENT: "spock60" # Default spock component name DEFAULT_BRANCH: "main" # Default branch for the spock component DEFAULT_CLEAN_FLAG: "false" # Default clean flag for scheduled runs @@ -13,7 +13,7 @@ on: workflow_dispatch: inputs: cli_branch: - description: "Select the CLI branch to build from (e.g. v25_STABLE)" + description: "Select the CLI branch to build from (e.g. main)" required: true default: "main" type: choice @@ -23,9 +23,9 @@ on: - REL24_10 component: - description: "Spock in-dev component to additionally build (e.g. spock50)" + description: "Spock in-dev component to additionally build (e.g. spock60)" required: true - default: "spock50" + default: "spock60" type: string branch: diff --git a/.github/workflows/current-arm9-daily-build-devel.yml b/.github/workflows/current-arm9-daily-build-devel.yml index f77a092d..b8c4d645 100644 --- a/.github/workflows/current-arm9-daily-build-devel.yml +++ b/.github/workflows/current-arm9-daily-build-devel.yml @@ -4,7 +4,7 @@ name: Current Daily Build Devel - arm9 env: DEFAULT_CLI_BRANCH: "main" # Default CLI branch for scheduled runs DEFAULT_MODE: "current" # Always "current" for this workflow - DEFAULT_COMPONENT: "spock50" # Default spock component name + DEFAULT_COMPONENT: "spock60" # Default spock component name DEFAULT_BRANCH: "main" # Default branch for the spock component DEFAULT_CLEAN_FLAG: "false" # Default clean flag for scheduled runs @@ -13,7 +13,7 @@ on: workflow_dispatch: inputs: cli_branch: - description: "Select the CLI branch to build from (e.g. v25_STABLE)" + description: "Select the CLI branch to build from (e.g. main)" required: true default: "main" type: choice @@ -23,9 +23,9 @@ on: - REL24_10 component: - description: "Spock in-dev component to additionally build (e.g. spock50)" + description: "Spock in-dev component to additionally build (e.g. spock60)" required: true - default: "spock50" + default: "spock60" type: string branch: diff --git a/build.sh b/build.sh index e0907f6a..f9a9e022 100755 --- a/build.sh +++ b/build.sh @@ -358,18 +358,21 @@ initPG () { initC "audit-pg$pgM" "audit" "$audit17V" "$outPlat" "postgres/audit" "" "" "nil" initC "hintplan-pg$pgM" "hintplan" "$hint17V" "$outPlat" "postgres/hintplan" "" "" "nil" initC "spock50-pg$pgM" "spock50" "$spock50V" "$outPlat" "postgres/spock50" "" "" "nil" + initC "spock60-pg$pgM" "spock60" "$spock60V" "$outPlat" "postgres/spock60" "" "" "nil" fi if [ "$pgM" == "16" ]; then initC "audit-pg$pgM" "audit" "$audit16V" "$outPlat" "postgres/audit" "" "" "nil" initC "hintplan-pg$pgM" "hintplan" "$hint16V" "$outPlat" "postgres/hintplan" "" "" "nil" initC "spock50-pg$pgM" "spock50" "$spock50V" "$outPlat" "postgres/spock50" "" "" "nil" + initC "spock60-pg$pgM" "spock60" "$spock60V" "$outPlat" "postgres/spock60" "" "" "nil" fi if [ "$pgM" == "15" ]; then initC "audit-pg$pgM" "audit" "$audit15V" "$outPlat" "postgres/audit" "" "" "nil" initC "hintplan-pg$pgM" "hintplan" "$hint15V" "$outPlat" "postgres/hintplan" "" "" "nil" initC "spock50-pg$pgM" "spock50" "$spock50V" "$outPlat" "postgres/spock50" "" "" "nil" + initC "spock60-pg$pgM" "spock60" "$spock60V" "$outPlat" "postgres/spock60" "" "" "nil" fi if [ "$pgM" == "15" ] || [ "$pgM" == "16" ] || [ "$pgM" == "17" ]; then diff --git a/env.sh b/env.sh index c7ee9792..76c3656f 100755 --- a/env.sh +++ b/env.sh @@ -8,6 +8,8 @@ bundle=pgedge api=pgedge ctlibsV=1.7 +spock60V=6.0.0-devel-1 + spock50V=5.0.0-1 spock40V=4.0.10-1 diff --git a/src/conf/versions.sql b/src/conf/versions.sql index 6c31e133..31f04ab5 100644 --- a/src/conf/versions.sql +++ b/src/conf/versions.sql @@ -84,6 +84,11 @@ INSERT INTO extensions VALUES ('spock50', 'spock', 1, 'spock', max_wal_senders=16 | hot_standby_feedback=on | wal_sender_timeout=5s | track_commit_timestamp=on | spock.conflict_resolution=last_update_wins | spock.save_resolutions=on | spock.conflict_log_level=DEBUG'); +INSERT INTO extensions VALUES ('spock60', 'spock', 1, 'spock', + 'wal_level=logical | max_worker_processes=12 | max_replication_slots=16 | + max_wal_senders=16 | hot_standby_feedback=on | wal_sender_timeout=5s | + track_commit_timestamp=on | spock.conflict_resolution=last_update_wins | + spock.save_resolutions=on | spock.conflict_log_level=DEBUG'); INSERT INTO extensions VALUES ('lolor', 'lolor', 0, '', ''); INSERT INTO extensions VALUES ('postgis', 'postgis', 1, 'postgis-3', ''); INSERT INTO extensions VALUES ('setuser', 'set_user', 1, 'set_user', ''); @@ -342,6 +347,15 @@ INSERT INTO versions VALUES ('spock50-pg15', '5.0.0-1', 'amd, arm', 1, '202507 INSERT INTO versions VALUES ('spock50-pg16', '5.0.0-1', 'amd, arm', 1, '20250715', 'pg16', '', ''); INSERT INTO versions VALUES ('spock50-pg17', '5.0.0-1', 'amd, arm', 1, '20250715', 'pg17', '', ''); +-- ## spock60 ########################### +INSERT INTO releases VALUES ('spock60-pg15', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); +INSERT INTO releases VALUES ('spock60-pg16', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); +INSERT INTO releases VALUES ('spock60-pg17', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); + +INSERT INTO versions VALUES ('spock60-pg15', '6.0.0-devel-1', 'amd, arm', 1, '20250801', 'pg15', '', ''); +INSERT INTO versions VALUES ('spock60-pg16', '6.0.0-devel-1', 'amd, arm', 1, '20250801', 'pg16', '', ''); +INSERT INTO versions VALUES ('spock60-pg17', '6.0.0-devel-1', 'amd, arm', 1, '20250801', 'pg17', '', ''); + -- ## LOLOR ############################# INSERT INTO projects VALUES ('lolor', 'pge', 4, 0, '', 1, 'https://github.com/pgedge/lolor/tags', 'spock', 1, 'spock.png', 'Logical Replication of Large Objects', 'https://github.com/pgedge/lolor/#spock', 'lola, lolah, kinks'); From e755133b8f010c63ee406060c083cea4e17070c7 Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Mon, 4 Aug 2025 08:16:59 -0500 Subject: [PATCH 20/25] move install url to downloads.pgedge.com (#361) --- cli/scripts/cluster.py | 2 +- cli/scripts/install.py | 2 +- devel/setup/compose/README.md | 6 +++--- devel/setup/compose/proxy_server.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index 766abe39..f0395bd5 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -15,7 +15,7 @@ BASE_DIR = "cluster" -DEFAULT_REPO = "https://pgedge-download.s3.amazonaws.com/REPO" +DEFAULT_REPO = "https://downloads.pgedge.com/platform/repos/download" def run_cmd( cmd, node, message, verbose, capture_output=False, ignore=False, important=False diff --git a/cli/scripts/install.py b/cli/scripts/install.py index 31225eba..1c402715 100644 --- a/cli/scripts/install.py +++ b/cli/scripts/install.py @@ -4,7 +4,7 @@ import sys, os, tarfile, platform VER = "25.1.0" -REPO = os.getenv("REPO", "https://pgedge-download.s3.amazonaws.com/REPO") +REPO = os.getenv("REPO", "https://downloads.pgedge.com/platform/repos/download") if sys.version_info < (3, 9): maj = sys.version_info.major diff --git a/devel/setup/compose/README.md b/devel/setup/compose/README.md index b21b2e0a..41ae6c30 100644 --- a/devel/setup/compose/README.md +++ b/devel/setup/compose/README.md @@ -63,9 +63,9 @@ If a local package does not exist, it will fallback to a package available in th The chosen repo corresponds to the URL that you choose to configure: -- http://repo:8000/download corresponds with https://pgedge-download.s3.amazonaws.com/REPO -- http://repo:8000/upstream corresponds with https://pgedge-upstream.s3.amazonaws.com/REPO -- http://repo:8000/devel corresponds with https://pgedge-devel.s3.amazonaws.com/REPO +- http://repo:8000/download corresponds with https://downloads.pgedge.com/platform/repos/download +- http://repo:8000/upstream corresponds with https://downloads.pgedge.com/platform/repos/upstream +- http://repo:8000/devel corresponds with https://downloads.pgedge.com/platform/repos/devel The `out` directory within the build container is mounted in the repo container to enable this setup. diff --git a/devel/setup/compose/proxy_server.py b/devel/setup/compose/proxy_server.py index 8a7de274..97b9b858 100644 --- a/devel/setup/compose/proxy_server.py +++ b/devel/setup/compose/proxy_server.py @@ -33,7 +33,7 @@ def do_GET(self): def proxy_request(self, repo, path): """Fetch content from the upstream server and send it to the client.""" - upstream_url = f"https://pgedge-{repo}.s3.amazonaws.com/REPO/{path}" + upstream_url = f"https://downloads.pgedge.com/platform/repos/{repo}/{path}" self.log_message("Proxying request to %s", upstream_url) try: with urllib.request.urlopen(upstream_url) as response: From c4cc10c3a36ed318017973c4d1620cc084dad250 Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Tue, 5 Aug 2025 16:45:58 +0500 Subject: [PATCH 21/25] Bump backrest version to 2.56.0 (#363) --- env.sh | 2 +- src/conf/versions.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/env.sh b/env.sh index 76c3656f..6ee75396 100755 --- a/env.sh +++ b/env.sh @@ -37,7 +37,7 @@ vectorV=0.8.0-1 bouncerV=1.23.1-1 catV=1.2.0 prompgexpV=0.15.0 -backrestV=2.53.1-1 +backrestV=2.56.0-1 wal2jV=2.6.0-1 citusV=12.1.5-1 diff --git a/src/conf/versions.sql b/src/conf/versions.sql index 31f04ab5..2c5c2804 100644 --- a/src/conf/versions.sql +++ b/src/conf/versions.sql @@ -438,8 +438,8 @@ INSERT INTO projects VALUES ('backrest', 'pge', 11, 0, '', 3, 'http://pgbackrest 'backrest', 0, 'backrest.png', 'Backup & Restore', 'http://pgbackrest.org', 'pg_backrest, pgbackrest'); INSERT INTO releases VALUES ('backrest', 2, 'backrest', 'pgBackRest', '', 'test', '', 1, 'MIT', 'EL', ''); -INSERT INTO versions VALUES ('backrest', '2.53.1-1', 'amd, arm', 1, '20240912', '', '', ''); -INSERT INTO versions VALUES ('backrest', '2.53-1', 'amd, arm', 0, '20240729', '', '', ''); +INSERT INTO versions VALUES ('backrest', '2.56.0-1', 'amd, arm', 1, '20240805', '', '', ''); +INSERT INTO versions VALUES ('backrest', '2.53.1-1', 'amd, arm', 0, '20240912', '', '', ''); -- ## PATRONI ########################### INSERT INTO projects VALUES ('patroni', 'app', 11, 0, '', 4, 'https://github.com/pgedge/pgedge-patroni/release', From a390bfaced42cf4ce8276446cb4bdf94b18752ac Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Wed, 30 Jul 2025 19:12:53 +0500 Subject: [PATCH 22/25] escape seq issue solved --- src/pgXX/init-pgXX.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/pgXX/init-pgXX.py b/src/pgXX/init-pgXX.py index 6111c7db..10737e80 100644 --- a/src/pgXX/init-pgXX.py +++ b/src/pgXX/init-pgXX.py @@ -62,7 +62,8 @@ def fatal_error(p_msg): if args.datadir == "": pg_data = os.path.join(data_root, pgver) else: - pg_data = args.datadir + + pg_data = args.datadir.replace(r'\ ', ' ') if not os.path.isdir(pg_data): os.makedirs(pg_data) @@ -195,12 +196,24 @@ def fatal_error(p_msg): util.update_postgresql_conf(pgver, i_port) -if util.get_platform() == "Linux": - os.system("cp " + pgver + "/genSelfCert.sh " + pg_data + "/.") - os.system(pg_data + "/genSelfCert.sh") +# ——— NEW: force Postgres to look in the right place for the certs ——— +conf_file = os.path.join(pg_data, "postgresql.conf") +ssl_block = """ +# — added by init script to enable SSL in data dir (quoting handles spaces) — +ssl = on +ssl_cert_file = '{0}/server.crt' +ssl_key_file = '{0}/server.key' +""".format(pg_data) -os.system("cp " + pgver + "/pg_hba.conf.nix " + pg_data + "/pg_hba.conf") +with open(conf_file, "a") as cf: + cf.write(ssl_block) + +# now generate your cert and copy pg_hba +if util.get_platform() == "Linux": + os.system(f'cp "{pgver}/genSelfCert.sh" "{pg_data}/."') + os.system(f'sh "{pg_data}/genSelfCert.sh"') +os.system(f'cp "{pgver}/pg_hba.conf.nix" "{pg_data}/pg_hba.conf"') if is_password: pg_pass_file = util.remember_pgpassword(pg_password, "*", "*", "*", os_user) else: From 18f21a0347b45c16a0f790131fa6c3abb3404528 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Wed, 30 Jul 2025 19:49:14 +0500 Subject: [PATCH 23/25] Update init-pgXX.py --- src/pgXX/init-pgXX.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pgXX/init-pgXX.py b/src/pgXX/init-pgXX.py index 10737e80..8d6d25f5 100644 --- a/src/pgXX/init-pgXX.py +++ b/src/pgXX/init-pgXX.py @@ -62,7 +62,6 @@ def fatal_error(p_msg): if args.datadir == "": pg_data = os.path.join(data_root, pgver) else: - pg_data = args.datadir.replace(r'\ ', ' ') if not os.path.isdir(pg_data): From d0eb01d1ffd3fa38e6a944ed77e4939f77021cc3 Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Tue, 5 Aug 2025 16:15:17 -0500 Subject: [PATCH 24/25] pass datadir as quoted --- cli/scripts/setup.py | 2 +- src/pgXX/init-pgXX.py | 27 ++++++++++----------------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/cli/scripts/setup.py b/cli/scripts/setup.py index 6e5452d8..d5a34632 100755 --- a/cli/scripts/setup.py +++ b/cli/scripts/setup.py @@ -104,7 +104,7 @@ def setup_pgedge(User=None, Passwd=None, dbName=None, port=None, pg_data=None, p util.exit_message( "pg_data cannot be set as relative path. Please specify absolute path instead" ) - pg_init_options = f"--datadir={pg_data}" + pg_init_options = f'--datadir="{pg_data}"' setup_core.check_pre_reqs( User, Passwd, dbName, port, pg_data, pg_major, pg_minor, spock_ver, autostart) diff --git a/src/pgXX/init-pgXX.py b/src/pgXX/init-pgXX.py index 8d6d25f5..9c983d40 100644 --- a/src/pgXX/init-pgXX.py +++ b/src/pgXX/init-pgXX.py @@ -5,6 +5,7 @@ import util, startup import argparse, os, sys, shutil, subprocess, getpass, json +import shlex MY_HOME = os.getenv("MY_HOME", "") @@ -62,7 +63,7 @@ def fatal_error(p_msg): if args.datadir == "": pg_data = os.path.join(data_root, pgver) else: - pg_data = args.datadir.replace(r'\ ', ' ') + pg_data = args.datadir if not os.path.isdir(pg_data): os.makedirs(pg_data) @@ -195,24 +196,16 @@ def fatal_error(p_msg): util.update_postgresql_conf(pgver, i_port) -# ——— NEW: force Postgres to look in the right place for the certs ——— -conf_file = os.path.join(pg_data, "postgresql.conf") -ssl_block = """ -# — added by init script to enable SSL in data dir (quoting handles spaces) — -ssl = on -ssl_cert_file = '{0}/server.crt' -ssl_key_file = '{0}/server.key' -""".format(pg_data) - -with open(conf_file, "a") as cf: - cf.write(ssl_block) - -# now generate your cert and copy pg_hba if util.get_platform() == "Linux": - os.system(f'cp "{pgver}/genSelfCert.sh" "{pg_data}/."') - os.system(f'sh "{pg_data}/genSelfCert.sh"') + gen_cert_src = os.path.join(pgver, "genSelfCert.sh") + gen_cert_dst = os.path.join(pg_data, "genSelfCert.sh") + os.system(f'cp {shlex.quote(gen_cert_src)} {shlex.quote(gen_cert_dst)}') + os.system(f'{shlex.quote(gen_cert_dst)}') + +pg_hba_src = os.path.join(pgver, "pg_hba.conf.nix") +pg_hba_dst = os.path.join(pg_data, "pg_hba.conf") +os.system(f'cp {shlex.quote(pg_hba_src)} {shlex.quote(pg_hba_dst)}') -os.system(f'cp "{pgver}/pg_hba.conf.nix" "{pg_data}/pg_hba.conf"') if is_password: pg_pass_file = util.remember_pgpassword(pg_password, "*", "*", "*", os_user) else: From 2a6a90b38377c741c5d9da8696ce8c63b3d4e2fb Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Tue, 5 Aug 2025 21:14:32 -0500 Subject: [PATCH 25/25] bump version to 25.2.0 (#364) --- cli/scripts/install.py | 2 +- cli/scripts/util.py | 2 +- env.sh | 4 ++-- src/conf/versions.sql | 3 ++- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cli/scripts/install.py b/cli/scripts/install.py index 1c402715..42861768 100644 --- a/cli/scripts/install.py +++ b/cli/scripts/install.py @@ -3,7 +3,7 @@ import sys, os, tarfile, platform -VER = "25.1.0" +VER = "25.2.0" REPO = os.getenv("REPO", "https://downloads.pgedge.com/platform/repos/download") if sys.version_info < (3, 9): diff --git a/cli/scripts/util.py b/cli/scripts/util.py index a8dc0ab2..da2cf20e 100644 --- a/cli/scripts/util.py +++ b/cli/scripts/util.py @@ -4,7 +4,7 @@ import os import time -MY_VERSION = "25.1.0" +MY_VERSION = "25.2.0" MY_CODENAME = "" DEFAULT_PG = "17" diff --git a/env.sh b/env.sh index 6ee75396..578e51ef 100755 --- a/env.sh +++ b/env.sh @@ -1,5 +1,5 @@ -hubV=25.1.0 -hubVV=25.1.0 +hubV=25.2.0 +hubVV=25.2.0 aceV=$hubV kirkV=$hubV diff --git a/src/conf/versions.sql b/src/conf/versions.sql index 2c5c2804..b7eb42c8 100644 --- a/src/conf/versions.sql +++ b/src/conf/versions.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS hub; CREATE TABLE hub(v TEXT NOT NULL PRIMARY KEY, c TEXT NOT NULL, d TEXT NOT NULL); -INSERT INTO hub VALUES ('25.1.0', '', '20250626'); +INSERT INTO hub VALUES ('25.2.0', '', '20250815'); DROP VIEW IF EXISTS v_versions; DROP VIEW IF EXISTS v_products; @@ -139,6 +139,7 @@ INSERT INTO projects VALUES ('hub', 'app', 0, 0, 'hub', 0, 'https://github.com/p INSERT INTO releases VALUES ('hub', 1, 'hub', '', '', 'hidden', '', 1, '', '', ''); INSERT INTO versions VALUES ('hub', (select v from hub), '', 1, (select d from hub), '', '', ''); +INSERT INTO versions VALUES ('hub', '25.1.0', '', 0, '20250626', '', '', ''); INSERT INTO versions VALUES ('hub', '25.0.0', '', 0, '20250603', '', '', ''); INSERT INTO versions VALUES ('hub', '24.10.13', '', 0, '20250509', '', '', ''); INSERT INTO versions VALUES ('hub', '24.10.11', '', 0, '20250224', '', '', '');