diff --git a/.github/workflows/current-amd8-daily-build-devel.yml b/.github/workflows/current-amd8-daily-build-devel.yml index 76a334ee..08b772be 100644 --- a/.github/workflows/current-amd8-daily-build-devel.yml +++ b/.github/workflows/current-amd8-daily-build-devel.yml @@ -4,7 +4,7 @@ name: Current Daily Build Devel - amd8 env: DEFAULT_CLI_BRANCH: "main" # Default CLI branch for scheduled runs DEFAULT_MODE: "current" # Always "current" for this workflow - DEFAULT_COMPONENT: "spock50" # Default spock component name + DEFAULT_COMPONENT: "spock60" # Default spock component name DEFAULT_BRANCH: "main" # Default branch for the spock component DEFAULT_CLEAN_FLAG: "false" # Default clean flag for scheduled runs @@ -13,7 +13,7 @@ on: workflow_dispatch: inputs: cli_branch: - description: "Select the CLI branch to build from (e.g. v25_STABLE)" + description: "Select the CLI branch to build from (e.g. main)" required: true default: "main" type: choice @@ -23,9 +23,9 @@ on: - REL24_10 component: - description: "Spock in-dev component to additionally build (e.g. spock50)" + description: "Spock in-dev component to additionally build (e.g. spock60)" required: true - default: "spock50" + default: "spock60" type: string branch: diff --git a/.github/workflows/current-arm9-daily-build-devel.yml b/.github/workflows/current-arm9-daily-build-devel.yml index f77a092d..b8c4d645 100644 --- a/.github/workflows/current-arm9-daily-build-devel.yml +++ b/.github/workflows/current-arm9-daily-build-devel.yml @@ -4,7 +4,7 @@ name: Current Daily Build Devel - arm9 env: DEFAULT_CLI_BRANCH: "main" # Default CLI branch for scheduled runs DEFAULT_MODE: "current" # Always "current" for this workflow - DEFAULT_COMPONENT: "spock50" # Default spock component name + DEFAULT_COMPONENT: "spock60" # Default spock component name DEFAULT_BRANCH: "main" # Default branch for the spock component DEFAULT_CLEAN_FLAG: "false" # Default clean flag for scheduled runs @@ -13,7 +13,7 @@ on: workflow_dispatch: inputs: cli_branch: - description: "Select the CLI branch to build from (e.g. v25_STABLE)" + description: "Select the CLI branch to build from (e.g. main)" required: true default: "main" type: choice @@ -23,9 +23,9 @@ on: - REL24_10 component: - description: "Spock in-dev component to additionally build (e.g. spock50)" + description: "Spock in-dev component to additionally build (e.g. spock60)" required: true - default: "spock50" + default: "spock60" type: string branch: diff --git a/build.sh b/build.sh index e0907f6a..f9a9e022 100755 --- a/build.sh +++ b/build.sh @@ -358,18 +358,21 @@ initPG () { initC "audit-pg$pgM" "audit" "$audit17V" "$outPlat" "postgres/audit" "" "" "nil" initC "hintplan-pg$pgM" "hintplan" "$hint17V" "$outPlat" "postgres/hintplan" "" "" "nil" initC "spock50-pg$pgM" "spock50" "$spock50V" "$outPlat" "postgres/spock50" "" "" "nil" + initC "spock60-pg$pgM" "spock60" "$spock60V" "$outPlat" "postgres/spock60" "" "" "nil" fi if [ "$pgM" == "16" ]; then initC "audit-pg$pgM" "audit" "$audit16V" "$outPlat" "postgres/audit" "" "" "nil" initC "hintplan-pg$pgM" "hintplan" "$hint16V" "$outPlat" "postgres/hintplan" "" "" "nil" initC "spock50-pg$pgM" "spock50" "$spock50V" "$outPlat" "postgres/spock50" "" "" "nil" + initC "spock60-pg$pgM" "spock60" "$spock60V" "$outPlat" "postgres/spock60" "" "" "nil" fi if [ "$pgM" == "15" ]; then initC "audit-pg$pgM" "audit" "$audit15V" "$outPlat" "postgres/audit" "" "" "nil" initC "hintplan-pg$pgM" "hintplan" "$hint15V" "$outPlat" "postgres/hintplan" "" "" "nil" initC "spock50-pg$pgM" "spock50" "$spock50V" "$outPlat" "postgres/spock50" "" "" "nil" + initC "spock60-pg$pgM" "spock60" "$spock60V" "$outPlat" "postgres/spock60" "" "" "nil" fi if [ "$pgM" == "15" ] || [ "$pgM" == "16" ] || [ "$pgM" == "17" ]; then diff --git a/build_all.sh b/build_all.sh index f584fbdf..3f8f5f22 100755 --- a/build_all.sh +++ b/build_all.sh @@ -13,7 +13,7 @@ if [ ! $num_p == "0" ] && [ ! $num_p == "1" ]; then fi if [ "$1" == "" ]; then - majorV=16 + majorV=17 echo "" echo "### Defaulting to pg $majorV ###" else diff --git a/cli/scripts/ace-tests/test_data_types.py b/cli/scripts/ace-tests/test_data_types.py index 950bf2dd..ff5d7aec 100644 --- a/cli/scripts/ace-tests/test_data_types.py +++ b/cli/scripts/ace-tests/test_data_types.py @@ -1,3 +1,6 @@ +from datetime import datetime, timedelta, date, time +from decimal import Decimal +from ipaddress import IPv4Address import pytest import psycopg import json @@ -31,50 +34,25 @@ def setup_datatypes(self, nodes): bytea_col BYTEA, point_col POINT, text_col TEXT, - text_array_col TEXT[] + text_array_col TEXT[], + bool_col BOOLEAN, + bigint_col BIGINT, + smallint_col SMALLINT, + numeric_col NUMERIC(10, 4), + real_col REAL, + time_col TIME, + date_col DATE, + timestamp_col TIMESTAMP, + interval_col INTERVAL, + inet_col INET, + macaddr_col MACADDR, + money_col MONEY ) """ ) # Insert sample data - cur.execute( - """ - INSERT INTO datatypes_test VALUES - ( - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - 42, - 3.14159, - ARRAY[1, 2, 3, 4, 5], - '{"key": "value", "nested": {"foo": "bar"}}', - decode('DEADBEEF', 'hex'), - point(1.5, 2.5), - 'sample text', - ARRAY['apple', 'banana', 'cherry'] - ), - ( - 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12', - 100, - 2.71828, - ARRAY[10, 20, 30], - '{"numbers": [1, 2, 3], "active": true}', - decode('BADDCAFE', 'hex'), - point(3.7, 4.2), - 'another sample', - ARRAY['dog', 'cat', 'bird'] - ), - ( - 'c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a13', - -17, - 0.577216, - ARRAY[]::INTEGER[], - '{"empty": true}', - NULL, - point(0, 0), - 'third sample', - ARRAY[]::TEXT[] - ) - """ - ) + self._insert_initial_data(cur) repset_add_datatypes_sql = """ SELECT spock.repset_add_table('test_repset', 'datatypes_test') @@ -106,6 +84,145 @@ def setup_datatypes(self, nodes): except Exception as e: pytest.fail(f"Failed to setup/cleanup datatypes test: {str(e)}") + def _insert_initial_data(self, cur): + """Helper method to insert the initial dataset.""" + cur.execute(self._get_initial_data_sql()) + + def _get_initial_data_sql(self): + """Returns the SQL for inserting the initial dataset.""" + return """ + INSERT INTO datatypes_test VALUES + ( + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + 42, + 3.14159, + ARRAY[1, 2, 3, 4, 5], + '{"key": "value", "nested": {"foo": "bar"}}', + decode('DEADBEEF', 'hex'), + point(1.5, 2.5), + 'sample text', + ARRAY['apple', 'banana', 'cherry'], + true, + 9223372036854775807, + 32767, + 12345.6789, + 123.456, + '12:34:56', + '2024-01-01', + '2024-01-01 12:34:56', + '30 days', + '192.168.1.1', + '08:00:2b:01:02:03', + 12345.67 + ), + ( + 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12', + 100, + 2.71828, + ARRAY[10, 20, 30], + '{"numbers": [1, 2, 3], "active": true}', + decode('BADDCAFE', 'hex'), + point(3.7, 4.2), + 'another sample', + ARRAY['dog', 'cat', 'bird'], + false, + -9223372036854775808, + -32768, + -12345.6789, + -123.456, + '23:59:59', + '2023-12-31', + '2023-12-31 23:59:59', + '-5 days', + '10.0.0.1', + '00:1A:2B:3C:4D:5E', + -12345.67 + ), + ( + 'c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a13', + -17, + 0.577216, + ARRAY[]::INTEGER[], + '{"empty": true}', + NULL, + point(0, 0), + 'third sample', + ARRAY[]::TEXT[], + true, + 0, + 0, + 0, + 0, + '00:00:00', + '1970-01-01', + '1970-01-01 00:00:00', + '0 seconds', + '0.0.0.0', + '00:00:00:00:00:00', + 0 + ), + ( + 'd0eebc99-9c0b-4ef8-bb6d-6bb9bd380a14', + NULL, + NULL, + ARRAY[NULL, 1, NULL], + NULL, + NULL, + NULL, + 'null', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ), + ( + 'e0eebc99-9c0b-4ef8-bb6d-6bb9bd380a15', + 1, + 'NaN', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ) + """ + + def reset_data(self, nodes): + """Reset data in the test table before each test function.""" + try: + for node in nodes: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("TRUNCATE TABLE datatypes_test") + self._insert_initial_data(cur) + conn.commit() + cur.close() + conn.close() + except Exception as e: + pytest.fail(f"Failed to reset data between tests: {str(e)}") + # Override the table_name parameter for all parameterized tests @pytest.mark.parametrize("table_name", ["public.datatypes_test"]) def test_simple_table_diff(self, cli, capsys, table_name): @@ -113,16 +230,28 @@ def test_simple_table_diff(self, cli, capsys, table_name): @pytest.mark.parametrize("table_name", ["public.datatypes_test"]) @pytest.mark.parametrize( - "column_name,test_value", + "column_name,test_value,expected_diffs", [ - ("int_col", "9999"), - ("float_col", "123.456"), - ("array_col", "ARRAY[99, 98, 97]"), - ("json_col", '\'{"test": "modified"}\''), - ("bytea_col", "decode('FEEDFACE', 'hex')"), - ("point_col", "point(99.9, 99.9)"), - ("text_col", "'modified-text'"), - ("text_array_col", "ARRAY['modified', 'text', 'array']"), + ("int_col", "9999", 5), + ("float_col", "123.456", 5), + ("array_col", "ARRAY[99, 98, 97]", 5), + ("json_col", '\'{"test": "modified"}\'', 5), + ("bytea_col", "decode('FEEDFACE', 'hex')", 5), + ("point_col", "point(99.9, 99.9)", 5), + ("text_col", "'modified-text'", 5), + ("text_array_col", "ARRAY['modified', 'text', 'array']", 5), + ("bool_col", "false", 5), + ("bigint_col", "1234567890123456789", 5), + ("smallint_col", "-32768", 5), + ("numeric_col", "98765.4321", 5), + ("real_col", "987.654", 5), + ("time_col", "'11:22:33'", 5), + ("date_col", "'2025-05-25'", 5), + ("timestamp_col", "'2025-05-25 11:22:33'", 5), + ("interval_col", "'90 days'", 5), + ("inet_col", "'192.168.100.200'", 5), + ("macaddr_col", "'01:23:45:67:89:ab'", 5), + ("money_col", "9876.54", 5), ], ) @pytest.mark.parametrize("key_column", ["id"]) @@ -134,6 +263,7 @@ def test_table_diff_with_differences( table_name, column_name, test_value, + expected_diffs, key_column, diff_file_path, ): @@ -154,11 +284,6 @@ def test_table_diff_with_differences( """ ) - modified_rows = cur.fetchall() - modified_indices = { - str(row[0]) for row in modified_rows - } # Convert UUID to string - conn.commit() cur.close() conn.close() @@ -185,55 +310,79 @@ def test_table_diff_with_differences( # Verify number of differences assert ( - len(diff_data["diffs"]["n1/n2"]["n2"]) == 3 - ), "Expected 3 differences," + len(diff_data["diffs"]["n1/n2"]["n2"]) == expected_diffs + ), f"Expected {expected_diffs} differences," f" found {len(diff_data['diffs']['n1/n2']['n2'])}" - # Verify modified rows are in diff - diff_indices = { - str(diff["id"]) for diff in diff_data["diffs"]["n1/n2"]["n2"] - } - assert ( - modified_indices == diff_indices - ), "Modified rows don't match diff file records" - # Verify the differences are correctly reported for diff in diff_data["diffs"]["n1/n2"]["n2"]: + diff_val = diff[column_name] + expected_val_str = test_value.strip("'") + if column_name == "json_col": assert ( - diff[column_name].get("test") == "modified" + diff_val.get("test") == "modified" ), f"Modified row {diff['id']} doesn't have expected JSON value" elif column_name == "array_col": - assert diff[column_name] == [ + assert diff_val == [ 99, 98, 97, ], f"Modified row {diff['id']} doesn't have expected array value" elif column_name == "text_array_col": - assert diff[column_name] == [ + assert diff_val == [ "modified", "text", "array", ], ( - f"Modified row {diff['id']} doesn't have expected " - "text array value" + f"Modified row {diff['id']} doesn't have expected text" + " array value" ) elif column_name == "point_col": assert ( - diff[column_name] == "(99.9,99.9)" + diff_val == "(99.9,99.9)" ), f"Modified row {diff['id']} doesn't have expected point value" elif column_name == "bytea_col": - print("bytea col: ", diff[column_name]) assert ( - diff[column_name] == "feedface" + diff_val == "feedface" ), f"Modified row {diff['id']} doesn't have expected bytea value" + elif column_name == "macaddr_col": + assert ( + diff_val == "01:23:45:67:89:ab" + ), f"Modified row {diff['id']} doesn't have expected macaddr value" + elif column_name == "money_col": + cleaned_diff = diff_val.replace("$", "").replace(",", "") + assert float(cleaned_diff) == float( + expected_val_str + ), f"Modified row {diff['id']} doesn't have expected money value" + elif column_name == "bool_col": + assert str(diff_val).lower() == expected_val_str.lower(), ( + f"Modified row {diff['id']} " + "doesn't have expected boolean value" + ) + elif column_name == "interval_col": + # Interval representation can vary, so we check equality + # directly in Postgres + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + "SELECT %s::interval = %s::interval", + (str(diff_val), expected_val_str), + ) + is_equal = cur.fetchone()[0] + cur.close() + conn.close() + assert is_equal, ( + f"Modified row {diff['id']} " + f"doesn't have expected interval value. " + f"Got {diff_val}, expected equivalence to {expected_val_str}" + ) else: - assert str(diff[column_name]) in ( - test_value.strip("'"), - "9999", - "123.456", - "modified-text", - ), f"Modified row {diff['id']} doesn't have expected value" + assert str(diff_val) == expected_val_str, ( + f"Modified row {diff['id']} " + f"doesn't have expected value, got {diff_val} " + f"expected {expected_val_str}" + ) except Exception as e: pytest.fail(f"Failed to test differences for {column_name}: {str(e)}") @@ -245,16 +394,28 @@ def test_simple_table_repair(self, cli, capsys, table_name, diff_file_path): @pytest.mark.parametrize("table_name", ["public.datatypes_test"]) @pytest.mark.parametrize("key_column", ["id"]) @pytest.mark.parametrize( - "column_name,test_value", + "column_name,test_value,expected_rerun_diffs", [ - ("int_col", "1234"), - ("float_col", "98.765"), - ("array_col", "ARRAY[11, 22, 33]"), - ("json_col", '\'{"rerun": "modified"}\''), - ("bytea_col", "decode('ABCDEF12', 'hex')"), - ("point_col", "point(88.8, 88.8)"), - ("text_col", "'rerun-modified'"), - ("text_array_col", "ARRAY['rerun', 'modified', 'array']"), + ("int_col", "1234", 5), + ("float_col", "98.765", 5), + ("array_col", "ARRAY[11, 22, 33]", 5), + ("json_col", '\'{"rerun": "modified"}\'', 5), + ("bytea_col", "decode('ABCDEF12', 'hex')", 5), + ("point_col", "point(88.8, 88.8)", 5), + ("text_col", "'rerun-modified'", 5), + ("text_array_col", "ARRAY['rerun', 'modified', 'array']", 5), + ("bool_col", "true", 5), + ("bigint_col", "-1234567890123456789", 5), + ("smallint_col", "32767", 5), + ("numeric_col", "-98765.4321", 5), + ("real_col", "-987.654", 5), + ("time_col", "'01:02:03'", 5), + ("date_col", "'2022-02-02'", 5), + ("timestamp_col", "'2022-02-02 01:02:03'", 5), + ("interval_col", "'60 days'", 5), + ("inet_col", "'127.0.0.1'", 5), + ("macaddr_col", "'fe:dc:ba:98:76:54'", 5), + ("money_col", "-9876.54", 5), ], ) def test_table_rerun_temptable( @@ -266,6 +427,7 @@ def test_table_rerun_temptable( key_column, column_name, test_value, + expected_rerun_diffs, diff_file_path, ): """Test table rerun temptable with various data types""" @@ -323,42 +485,115 @@ def test_table_rerun_temptable( diff_data = json.load(f) assert ( - len(diff_data["diffs"]["n1/n2"]["n2"]) == 3 - ), f"Expected 3 differences, found {len(diff_data['diffs']['n1/n2']['n2'])}" + len(diff_data["diffs"]["n1/n2"]["n2"]) == expected_rerun_diffs + ), f"Expected {expected_rerun_diffs} differences, " + f"found {len(diff_data['diffs']['n1/n2']['n2'])}" # Verify the differences are correctly reported for diff in diff_data["diffs"]["n1/n2"]["n2"]: + diff_val = diff[column_name] + expected_val_str = test_value.strip("'") + if column_name == "json_col": assert ( - diff[column_name].get("rerun") == "modified" + diff_val.get("rerun") == "modified" ), f"Modified row {diff['id']} doesn't have expected JSON value" elif column_name == "array_col": - assert diff[column_name] == [ + assert diff_val == [ 11, 22, 33, ], f"Modified row {diff['id']} doesn't have expected array value" elif column_name == "text_array_col": - assert diff[column_name] == [ + assert diff_val == [ "rerun", "modified", "array", ], f"Modified row {diff['id']} doesn't have expected text array value" elif column_name == "point_col": assert ( - diff[column_name] == "(88.8,88.8)" + diff_val == "(88.8,88.8)" ), f"Modified row {diff['id']} doesn't have expected point value" elif column_name == "bytea_col": assert ( - diff[column_name] == "abcdef12" + diff_val == "abcdef12" ), f"Modified row {diff['id']} doesn't have expected bytea value" + elif column_name == "macaddr_col": + assert ( + diff_val == "fe:dc:ba:98:76:54" + ), f"Modified row {diff['id']} doesn't have expected macaddr value" + elif column_name == "money_col": + cleaned_diff = diff_val.replace("$", "").replace(",", "") + assert float(cleaned_diff) == float( + expected_val_str + ), f"Modified row {diff['id']} doesn't have expected money value" + elif column_name == "bool_col": + assert ( + str(diff_val).lower() == expected_val_str.lower() + ), f"Modified row {diff['id']} doesn't have expected boolean value" + elif column_name == "interval_col": + # Interval representation can vary, so we check equality in the DB + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + "SELECT %s::interval = %s::interval", + (str(diff_val), expected_val_str), + ) + is_equal = cur.fetchone()[0] + cur.close() + conn.close() + assert is_equal, ( + f"Modified row {diff['id']} " + f"doesn't have expected interval value. " + f"Got {diff_val}, expected equivalence to {expected_val_str}" + ) else: - assert str(diff[column_name]) in ( - test_value.strip("'"), - "1234", - "98.765", - "rerun-modified", - ), f"Modified row {diff['id']} doesn't have expected value" + assert str(diff_val) == expected_val_str, ( + f"Modified row {diff['id']} " + f"doesn't have expected value, got {diff_val} " + f"expected {expected_val_str}" + ) + + def _verify_repaired_value(self, column_name, repaired_value, expected_value): + """Helper function to verify repaired values based on data type""" + if column_name == "bytea_col": + assert ( + repaired_value == expected_value + ), "Repaired bytea value doesn't match expected value" + elif column_name == "point_col": + if isinstance(repaired_value, str): + repaired_tuple = tuple( + map(float, repaired_value.strip("()").split(",")) + ) + else: + repaired_tuple = repaired_value + + expected_tuple = tuple(map(float, expected_value.strip("()").split(","))) + assert ( + repaired_tuple == expected_tuple + ), "Repaired point value doesn't match expected value" + elif column_name == "numeric_col": + assert repaired_value == Decimal( + expected_value + ), f"Repaired value for {column_name} doesn't match" + elif column_name == "inet_col": + assert repaired_value == IPv4Address( + expected_value + ), f"Repaired value for {column_name} doesn't match" + elif column_name in ["time_col", "date_col", "timestamp_col", "interval_col"]: + assert ( + repaired_value == expected_value + ), f"Repaired value for {column_name} doesn't match" + elif column_name == "money_col": + cleaned_repaired = str(repaired_value).replace("$", "").replace(",", "") + cleaned_expected = str(expected_value).replace("$", "").replace(",", "") + assert float(cleaned_repaired) == float( + cleaned_expected + ), f"Repaired money value doesn't match for {column_name}" + else: + assert ( + repaired_value == expected_value + ), f"Repaired value doesn't match expected value for {column_name}" @pytest.mark.parametrize("table_name", ["public.datatypes_test"]) @pytest.mark.parametrize( @@ -376,6 +611,22 @@ def test_table_rerun_temptable( "ARRAY['modified', 'text', 'array']", ["modified", "text", "array"], ), + ("bool_col", "false", False), + ("bigint_col", "1234567890123456789", 1234567890123456789), + ("smallint_col", "-32768", -32768), + ("numeric_col", "98765.4321", Decimal("98765.4321")), + ("real_col", "987.654", 987.654), + ("time_col", "'11:22:33'", time(11, 22, 33)), + ("date_col", "'2025-05-25'", date(2025, 5, 25)), + ( + "timestamp_col", + "'2025-05-25 11:22:33'", + datetime(2025, 5, 25, 11, 22, 33), + ), + ("interval_col", "'90 days'", timedelta(days=90)), + ("inet_col", "'192.168.100.200'", "192.168.100.200"), + ("macaddr_col", "'01:23:45:67:89:ab'", "01:23:45:67:89:ab"), + ("money_col", "9876.54", "$9,876.54"), ], ) def test_table_repair_datatypes( @@ -441,18 +692,177 @@ def test_table_repair_datatypes( conn.close() # Compare with expected value - if column_name == "bytea_col": - assert ( - repaired_value == expected_value - ), "Repaired bytea value doesn't match expected value" - elif column_name == "point_col": - assert ( - str(repaired_value) == expected_value - ), "Repaired point value doesn't match expected value" - else: - assert ( - repaired_value == expected_value - ), f"Repaired value doesn't match expected value for {column_name}" + self._verify_repaired_value(column_name, repaired_value, expected_value) except Exception as e: pytest.fail(f"Test failed: {str(e)}") + + @pytest.mark.parametrize("id_to_update", ["d0eebc99-9c0b-4ef8-bb6d-6bb9bd380a14"]) + def test_null_and_string_literal_handling( + self, cli, capsys, diff_file_path, id_to_update + ): + """ + Verify that NULL values and string literals like 'null' are + handled correctly. + """ + try: + # Our prior repair unfortunately reset a lot of fields, so we reset + # the data first here + self.reset_data(nodes=["n1", "n2"]) + + # On n2, update text_col from 'null' to 'not null' and + # int_col from NULL to a number + conn = psycopg.connect(host="n2", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + # This specific id has "null" as a literal in the text_col + cur.execute( + """ + UPDATE datatypes_test + SET text_col = 'not null anymore', int_col = 123 + WHERE id = %s + """, + (id_to_update,), + ) + conn.commit() + cur.close() + conn.close() + + # Run table-diff + cli.table_diff(cluster_name="eqn-t9da", table_name="public.datatypes_test") + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file_path.path = match.group(1) + + # Verify diff file + with open(diff_file_path.path, "r") as f: + diff_data = json.load(f) + + diffs_n1 = diff_data["diffs"]["n1/n2"]["n1"] + diffs_n2 = diff_data["diffs"]["n1/n2"]["n2"] + + assert len(diffs_n1) == 1, "Expected 1 difference on n1" + assert len(diffs_n2) == 1, "Expected 1 difference on n2" + + # Check n1 (original values) + assert diffs_n1[0]["id"] == id_to_update + assert diffs_n1[0]["text_col"] == "null" + assert diffs_n1[0]["int_col"] is None + + # Check n2 (modified values) + assert diffs_n2[0]["id"] == id_to_update + assert diffs_n2[0]["text_col"] == "not null anymore" + assert diffs_n2[0]["int_col"] == 123 + + # Run table-repair + cli.table_repair( + cluster_name="eqn-t9da", + table_name="public.datatypes_test", + diff_file=diff_file_path.path, + source_of_truth="n2", + ) + + # Verify repair on n1 + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + """ + SELECT text_col, int_col FROM datatypes_test + WHERE id = %s + """, + (id_to_update,), + ) + repaired_text, repaired_int = cur.fetchone() + cur.close() + conn.close() + + assert repaired_text == "not null anymore" + assert repaired_int == 123 + + except Exception as e: + pytest.fail(f"Test for null handling failed: {str(e)}") + + @pytest.mark.parametrize("id_to_update", ["e0eebc99-9c0b-4ef8-bb6d-6bb9bd380a15"]) + def test_ast_literal_eval_fallback(self, cli, capsys, diff_file_path, id_to_update): + """ + Verify that the fallback to string representation works when + ast.literal_eval fails. + """ + try: + # Resetting again here + self.reset_data(nodes=["n1", "n2"]) + + # On n2, update float_col from NaN to a valid number + conn = psycopg.connect(host="n2", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + cur.execute( + """ + UPDATE datatypes_test + SET float_col = 1.23 + WHERE id = %s + """, + (id_to_update,), + ) + conn.commit() + cur.close() + conn.close() + + # Run table-diff + cli.table_diff(cluster_name="eqn-t9da", table_name="public.datatypes_test") + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file_path.path = match.group(1) + + # Verify diff file + with open(diff_file_path.path, "r") as f: + diff_data = json.load(f) + + diffs_n1 = diff_data["diffs"]["n1/n2"]["n1"] + diffs_n2 = diff_data["diffs"]["n1/n2"]["n2"] + + assert len(diffs_n1) == 1, "Expected 1 difference on n1" + assert len(diffs_n2) == 1, "Expected 1 difference on n2" + + # Check n1 (original 'NaN' value) + assert diffs_n1[0]["id"] == id_to_update + assert diffs_n1[0]["float_col"] == "nan" + + # Check n2 (modified value) + assert diffs_n2[0]["id"] == id_to_update + assert diffs_n2[0]["float_col"] == 1.23 + + # Run table-repair + cli.table_repair( + cluster_name="eqn-t9da", + table_name="public.datatypes_test", + diff_file=diff_file_path.path, + source_of_truth="n2", + ) + + # Verify repair on n1 + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + """ + SELECT float_col FROM datatypes_test + WHERE id = %s + """, + (id_to_update,), + ) + repaired_float = cur.fetchone()[0] + cur.close() + conn.close() + + assert repaired_float == 1.23 + + except Exception as e: + pytest.fail(f"Test for ast.literal_eval fallback failed: {str(e)}") diff --git a/cli/scripts/ace.py b/cli/scripts/ace.py index ecf50d43..0fed3a05 100755 --- a/cli/scripts/ace.py +++ b/cli/scripts/ace.py @@ -727,63 +727,47 @@ def check_diff_file_format(diff_file_path: str, task) -> dict: return diff_json -def convert_pg_type_to_json(item: str, type: str): +def convert_pg_type_to_json(item, type): """ Converts a value from a postgres column to a json-compatible type. """ # TODO: Need to revisit this. - try: - # List of types that should be treated as strings - string_types = [ - "char", - "text", - "time", - "bytea", - "uuid", - "date", - "timestamp", - "interval", - "inet", - "macaddr", - "xml", - "money", - "point", - "line", - "polygon", - ] - - # Types that can be directly represented in JSON - json_compatible_types = [ - "json", - "jsonb", - "boolean", - "integer", - "bigint", - "smallint", - "numeric", - "real", - "double precision", - ] - - type_lower = type.lower() - - if not item or item == "" or item.lower() == "null" or item.lower() == "none": - return None - elif "[]" in type_lower: - return ast.literal_eval(item) - elif any(s in type_lower for s in json_compatible_types): - # For JSON-compatible types, parse them using AST - return ast.literal_eval(item) - elif any(s in type_lower for s in string_types): - return item - else: - # Default to treating as string if type is unknown - return item + type_lower = type.lower() - except Exception as e: - raise AceException( - f"Could not convert value {item} to {type} while writing to json: {e}" - ) + # Types that should be parsed into native JSON types (not strings) + json_compatible_types = [ + "json", + "jsonb", + "boolean", + "integer", + "bigint", + "smallint", + "numeric", + "real", + "double precision", + ] + + is_parsable = ( + any(s in type_lower for s in json_compatible_types) or "[]" in type_lower + ) + + if not is_parsable: + # For string-like types (text, varchar, etc.), we return the value + # directly. This correctly preserves string literals like 'None' or + # 'null'. A database NULL would arrive here as item=None from the driver. + return item + + # For parsable types (numeric, boolean, json, array), we can interpret + # 'null' and 'none'. + if item is None or str(item).lower() in ("", "null", "none"): + return None + + try: + # For JSON-compatible types, parse them using AST + return ast.literal_eval(str(item)) + except (ValueError, SyntaxError): + # If conversion fails, treat as a string + return str(item) def convert_json_to_pg_type(rows, cols_list, col_types) -> list[tuple]: @@ -841,31 +825,40 @@ def convert_json_to_pg_type(rows, cols_list, col_types) -> list[tuple]: modified_row = tuple() for col_name in cols_list: col_type = col_types[col_name] - elem = str(row[col_name]) + elem = row[col_name] + type_lower = col_type.lower() try: - type_lower = col_type.lower() - - if ( - not elem - or elem == "" - or elem.lower() == "null" - or elem.lower() == "none" - ): - modified_row += (None,) - elif "[]" in type_lower: - modified_row += (ast.literal_eval(elem),) - elif any(s in type_lower for s in string_types): + # If the column type is a string type, we don't need to do anything + # special. A value of None will be converted to NULL by psycopg. + if any(s in type_lower for s in string_types): if type_lower == "bytea": - modified_row += (bytes.fromhex(elem),) + # We stored bytea as hex, so we need to convert it back + if elem is not None: + modified_row += (bytes.fromhex(elem),) + else: + modified_row += (None,) else: modified_row += (elem,) + continue + + # For non-string types, if the value is None, or looks like null, + # it should be treated as such. + if elem is None or str(elem).lower() in ("null", "none", ""): + modified_row += (None,) + continue + + elem_str = str(elem) + + if "[]" in type_lower: + modified_row += (ast.literal_eval(elem_str),) elif any(s in type_lower for s in json_compatible_types): - item = ast.literal_eval(elem) - if type_lower == "jsonb" or type_lower == "json": + item = ast.literal_eval(elem_str) + if type_lower in ("jsonb", "json"): item = json.dumps(item) modified_row += (item,) else: + # Fallback for any other types modified_row += (elem,) except (ValueError, SyntaxError): diff --git a/cli/scripts/ace_config.py b/cli/scripts/ace_config.py index 1580b1c4..39379e0e 100644 --- a/cli/scripts/ace_config.py +++ b/cli/scripts/ace_config.py @@ -12,6 +12,9 @@ STATEMENT_TIMEOUT = 0 # in milliseconds CONNECTION_TIMEOUT = 10 # in seconds +# Whether to use repeatable read isolation for Merkle tree updates +USE_REPEATABLE_READ = False + # Default values for ACE table-diff MAX_DIFF_ROWS = 1_000_000 MIN_DIFF_BLOCK_SIZE = 1000 diff --git a/cli/scripts/ace_core.py b/cli/scripts/ace_core.py index 91567668..b0c35b45 100644 --- a/cli/scripts/ace_core.py +++ b/cli/scripts/ace_core.py @@ -574,12 +574,16 @@ def compare_checksums(worker_id, shared_objects, worker_state, pkey1, pkey2): for row_key in t1_only: worker_diffs[node_pair_key][host1].append( - dict(zip(cols, (str(x) for x in row_key))) + dict( + zip(cols, (str(x) if x is not None else None for x in row_key)) + ) ) for row_key in t2_only: worker_diffs[node_pair_key][host2].append( - dict(zip(cols, (str(x) for x in row_key))) + dict( + zip(cols, (str(x) if x is not None else None for x in row_key)) + ) ) total_diffs += max(len(t1_only), len(t2_only)) diff --git a/cli/scripts/ace_mtree.py b/cli/scripts/ace_mtree.py index a621b66b..32eb7980 100644 --- a/cli/scripts/ace_mtree.py +++ b/cli/scripts/ace_mtree.py @@ -1025,7 +1025,6 @@ def split_blocks(conn, schema, table, key, blocks, block_size): i += 1 - conn.commit() pbar.close() return list(modified_positions) @@ -1399,7 +1398,6 @@ def merge_blocks(conn, schema, table, key, blocks, block_size): if i >= len(blocks): break - conn.commit() pbar.close() return list(modified_positions) @@ -1408,7 +1406,7 @@ def update_mtree(mtree_task: MerkleTreeTask, skip_all_checks=False) -> None: """ Update a Merkle tree by recomputing hashes for dirty leaf nodes and new blocks. Also processes any pending block rebalancing operations. - Uses repeatable read isolation to ensure consistency during the update. + Uses repeatable read isolation if config.USE_REPEATABLE_READ is True. Args: cluster_name (str): Name of the cluster @@ -1455,7 +1453,8 @@ def update_mtree(mtree_task: MerkleTreeTask, skip_all_checks=False) -> None: for node in mtree_task.fields.cluster_nodes: _, conn = mtree_task.connection_pool.connect(node) - conn.set_isolation_level(IsolationLevel.REPEATABLE_READ) + if config.USE_REPEATABLE_READ: + conn.set_isolation_level(IsolationLevel.REPEATABLE_READ) print(f"\nUpdating Merkle tree on node: {node['name']}") @@ -1476,7 +1475,6 @@ def update_mtree(mtree_task: MerkleTreeTask, skip_all_checks=False) -> None: if not blocks_to_update: print(f"No updates needed for {node['name']}") - conn.commit() continue # First identify blocks that might need splitting based on insert count @@ -1557,7 +1555,6 @@ def update_mtree(mtree_task: MerkleTreeTask, skip_all_checks=False) -> None: if not blocks_to_update: print(f"No updates needed for {node['name']}") - conn.commit() continue print(f"Found {len(blocks_to_update)} blocks to update") diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index a4915c4a..f0395bd5 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -6,14 +6,16 @@ import meta import time import sys +import setup_core import getpass from tabulate import tabulate # type: ignore -from ipaddress import ip_address +import socket import os import re + BASE_DIR = "cluster" -DEFAULT_REPO = "https://pgedge-download.s3.amazonaws.com/REPO" +DEFAULT_REPO = "https://downloads.pgedge.com/platform/repos/download" def run_cmd( cmd, node, message, verbose, capture_output=False, ignore=False, important=False @@ -1063,12 +1065,14 @@ def get_cluster_info(cluster_name): ) try: if public_ip: - ip_address(public_ip) + socket.gethostbyname(public_ip) + if private_ip: - ip_address(private_ip) - except ValueError: + socket.gethostbyname(private_ip) + + except socket.gaierror as e: validation_errors.append( - f"Invalid IP address provided for node {node.get('name')}." + f"Error resolving hostname or ip adress for node {node.get('name')} : {e}." ) for sub_node in node.get("sub_nodes", []): @@ -1098,12 +1102,12 @@ def get_cluster_info(cluster_name): ) try: if public_ip: - ip_address(public_ip) + socket.gethostbyname(public_ip) if private_ip: - ip_address(private_ip) - except ValueError: + socket.gethostbyname(private_ip) + except socket.gaierror as e: validation_errors.append( - f"Invalid IP address provided for sub-node {sub_node.get('name')}." + f"Error resolving hostname or ip adress for sub-node {sub_node.get('name')}: {e}." ) if validation_errors: @@ -1312,7 +1316,9 @@ def init(cluster_name, install=True): parsed_json = get_cluster_json(cluster_name) if parsed_json is None: util.exit_message("Unable to load cluster JSON", 1) - + pg_version = db_settings["pg_version"] + spock_ver = db_settings.get("spock_version", util.get_default_spock(pg_version)) + util.validate_spock_pg_compat(spock_ver, pg_version) verbose = parsed_json.get("log_level", "info") all_nodes = nodes.copy() diff --git a/cli/scripts/db.py b/cli/scripts/db.py index f5138bcc..9f9bf0f5 100755 --- a/cli/scripts/db.py +++ b/cli/scripts/db.py @@ -8,6 +8,8 @@ import json import util import fire +import re +import psycopg def create(db=None, User=None, Passwd=None, pg=None, spock=None, help=False): """ @@ -116,11 +118,15 @@ def guc_set(guc_name, guc_value): nc = "./pgedge " ncb = nc + "pgbin " + str(pg) + " " - cmd = f"ALTER SYSTEM SET {guc_name} = {guc_value}" + guc_value = re.sub(r'([\'\\])', r'\1\1', str(guc_value)) + cmd = f"ALTER SYSTEM SET {guc_name} = '{guc_value}'" + rc1 = util.echo_cmd(ncb + '"psql -q -c \\"' + cmd + '\\" postgres"',False) cmd = f"SELECT pg_reload_conf()" rc2 = util.echo_cmd(ncb + '"psql -q -c \\"' + cmd + '\\" postgres"',False) + rcs = rc1 + rc2 + if rcs == 0: util.message(f"Set GUC {guc_name} to {guc_value}","info") else: diff --git a/cli/scripts/install.py b/cli/scripts/install.py index 31225eba..42861768 100644 --- a/cli/scripts/install.py +++ b/cli/scripts/install.py @@ -3,8 +3,8 @@ import sys, os, tarfile, platform -VER = "25.1.0" -REPO = os.getenv("REPO", "https://pgedge-download.s3.amazonaws.com/REPO") +VER = "25.2.0" +REPO = os.getenv("REPO", "https://downloads.pgedge.com/platform/repos/download") if sys.version_info < (3, 9): maj = sys.version_info.major diff --git a/cli/scripts/meta.py b/cli/scripts/meta.py index b399e43a..9cd5fb1c 100644 --- a/cli/scripts/meta.py +++ b/cli/scripts/meta.py @@ -306,7 +306,6 @@ def get_default_spock(pgv): + pgv + "' \n" + " AND component LIKE 'spock%'" - + " AND version not LIKE '%devel%'" ) try: c = con.cursor() diff --git a/cli/scripts/setup.py b/cli/scripts/setup.py index 2fc21aa9..d5a34632 100755 --- a/cli/scripts/setup.py +++ b/cli/scripts/setup.py @@ -95,7 +95,8 @@ def setup_pgedge(User=None, Passwd=None, dbName=None, port=None, pg_data=None, p pg_ver = df_pg pg_major, pg_minor = setup_core.parse_pg(pg_ver) - + # Validate that if Spock ≥5 we're using PG 15.13+, 16.9+ or 17.5+Add commentMore actions + util.validate_spock_pg_compat(spock_ver, pg_ver) pg_init_options = "" if pg_data is not None: pg_data = pg_data.rstrip("/") @@ -103,7 +104,7 @@ def setup_pgedge(User=None, Passwd=None, dbName=None, port=None, pg_data=None, p util.exit_message( "pg_data cannot be set as relative path. Please specify absolute path instead" ) - pg_init_options = f"--datadir={pg_data}" + pg_init_options = f'--datadir="{pg_data}"' setup_core.check_pre_reqs( User, Passwd, dbName, port, pg_data, pg_major, pg_minor, spock_ver, autostart) diff --git a/cli/scripts/spock.py b/cli/scripts/spock.py index 3376b66c..3bec2756 100644 --- a/cli/scripts/spock.py +++ b/cli/scripts/spock.py @@ -198,36 +198,6 @@ def node_drop(node_name, db): sys.exit(0) -def node_alter_location(node_name, location, db): - """Set location details for spock node.""" - pg_v,spock_v = get_spock_ver() - - [location_nm, country, state, lattitude, longitude] = util.get_location_dtls( - location - ) - - sql = """ -UPDATE spock.node - SET location_nm = ?, country = ?, state = ?, lattitude = ?, longitude = ? - WHERE location = ? -""" - - con = util.get_pg_connection(pg_v, db, util.get_user()) - - rc = 0 - try: - con = util.get_pg_connection(pg_v, "postgres", util.get_user()) - cur = con.cursor(row_factory=psycopg.rows.dict_row) - cur.execute(sql, [location_nm, country, state, lattitude, longitude]) - con.commit() - except Exception as e: - util.print_exception(e) - con.rollback() - rc = 1 - - sys.exit(rc) - - def node_list(db): """Display node table. @@ -726,7 +696,7 @@ def set_readonly(readonly="off"): util.message("spock.set_readonly() deprecated, use db.set_readonly() instead", "warning") - return(db.set_readonly(readonly, pg)) + return(db.set_readonly(readonly)) def get_pii_cols(db, schema=None): @@ -1017,7 +987,6 @@ def metrics_check(db): { "node-create": node_create, "node-drop": node_drop, - "node-alter-location": node_alter_location, "node-list": node_list, "node-add-interface": node_add_interface, "node-drop-interface": node_drop_interface, diff --git a/cli/scripts/um.py b/cli/scripts/um.py index f07aedbc..ceff8e3a 100755 --- a/cli/scripts/um.py +++ b/cli/scripts/um.py @@ -3,8 +3,9 @@ import os, sys, glob, sqlite3, time import fire, meta, util - isJSON = util.isJSON +import re +import sqlite3 as _sqlite3 MY_HOME = util.MY_HOME @@ -83,20 +84,29 @@ def update(): run_cmd("update") + +# Match both "spock50" and "spock50-pg17" (and variants like spock5, spock5-pg16) +_SPOCK5_NAME_RE = re.compile(r"^spock5(?:0)?(?:-pg\d+)?$", re.IGNORECASE) +_SPOCK5_VER_RE = re.compile(r"^5\.", re.IGNORECASE) +_SPOCK50_RE = re.compile(r"^spock50(?:-pg\d+)?$", re.IGNORECASE) + def install(component, active=True): """Install a component.""" + # Trigger pre-check ONLY for Spock 5.0 artifacts + if _SPOCK50_RE.match(component): + validate_spock_upgrade() # should sys.exit(1) on failure; otherwise just returns + + # Common path (no duplication) if active not in (True, False): util.exit_message("'active' parm must be True or False") - - cmd = "install" - if active is False: - cmd = "install --no-preload" + cmd = "install" if active else "install --no-preload" util.message(f"um.install({cmd} {component})", "debug") run_cmd(cmd, component) + def remove(component): """Uninstall a component.""" installed_comp_list = meta.get_component_list() @@ -234,6 +244,76 @@ def verify_metadata(Project="", Stage="prod", IsCurrent=0): meta.pretty_sql(sql) +def validate_spock_upgrade(): + """ + Validate Spock↔PostgreSQL compatibility for an upcoming Spock 5 install. + """ + + DB_PATH = "data/conf/db_local.db" + + SPOCK_SQL = """ + SELECT version AS spock_ver, component AS spock_comp + FROM components + WHERE component LIKE 'spock%' + ORDER BY CASE + WHEN version LIKE '5.%' THEN 5 + WHEN version LIKE '4.%' THEN 4 + ELSE 0 + END DESC, + version DESC + LIMIT 1; + """ + + PG_SQL = """ + SELECT version AS pg_ver + FROM components + WHERE component LIKE 'pg__' + ORDER BY CAST(substr(component, 3) AS INTEGER) DESC + LIMIT 1; + """ + + try: + with _sqlite3.connect(DB_PATH) as conn: + sp_row = conn.execute(SPOCK_SQL).fetchone() + pg_row = conn.execute(PG_SQL).fetchone() + except _sqlite3.Error as err: + sys.exit(f"ERROR: SQLite query failed: {err}") + + if not pg_row: + sys.exit("ERROR: No PostgreSQL version row found.") + + pg_ver = pg_row[0] + spock_ver, spock_comp = (sp_row or (None, None)) + + # Already on Spock 5? No-op. + if spock_ver and ( + _SPOCK5_NAME_RE.match(spock_comp) or _SPOCK5_VER_RE.match(spock_ver) + ): + return 0 + + # Downtime warning + banner = "=" * 80 + + if not spock_ver: + # Case 1: no existing Spock installed, run validation but don't print banner + try: + util.validate_spock_pg_compat('50', pg_ver) + except Exception as exc: + sys.exit(f"ERROR: Compatibility check failed: {exc}") + + elif spock_ver.startswith('4'): + # Case 2: Spock 4.x installed, this is a major upgrade + print(f"\n{banner}") + print("*** WARNING: This operation will cause downtime! ***") + print(f"{banner}\n") + print(f"Detected Spock version {spock_ver} on PostgreSQL {pg_ver}") + try: + util.validate_spock_pg_compat(spock_ver, pg_ver) + except Exception as exc: + sys.exit(f"ERROR: Compatibility check failed: {exc}") + + print("Compatibility check passed.") + return 0 if __name__ == "__main__": fire.Fire( diff --git a/cli/scripts/util.py b/cli/scripts/util.py index b4335a21..da2cf20e 100644 --- a/cli/scripts/util.py +++ b/cli/scripts/util.py @@ -4,12 +4,12 @@ import os import time -MY_VERSION = "25.1.0" +MY_VERSION = "25.2.0" MY_CODENAME = "" -DEFAULT_PG = "16" -DEFAULT_SPOCK = "40" -DEFAULT_SPOCK_17 = "40" +DEFAULT_PG = "17" +DEFAULT_SPOCK = "50" +DEFAULT_SPOCK_17 = "50" MY_CMD = os.getenv("MY_CMD", None) MY_HOME = os.getenv("MY_HOME", None) MY_LIBS = f"{MY_HOME}/hub/scripts/lib" @@ -37,6 +37,7 @@ import subprocess import getpass import filecmp +import re from subprocess import Popen, PIPE, STDOUT from datetime import datetime, timedelta from urllib import request as urllib2 @@ -124,6 +125,121 @@ def get_default_spock(pgv): return(DEFAULT_SPOCK) + +def validate_spock_pg_compat(spock_ver: str = None, pg_ver: str = None) -> None: + """ + Compatibility rules: + • If Spock < 5.0.0 ⇒ works with any supported PostgreSQL major. + • If Spock ≥ 5.0.0 ⇒ + – PG15 must be ≥ 15.13 + – PG16 must be ≥ 16.9 + – PG17 must be ≥ 17.5 + + Also supports shorthand Spock strings: + – "50" → "5.0.0", "40" → "4.0.0", etc. + """ + # 0) Fill in defaults if user didn’t pass anything + if not pg_ver: + pg_ver = DEFAULT_PG + else: + pg_ver = str(pg_ver) # ← force to string + + if not spock_ver: + maj = int(pg_ver.split(".", 1)[0]) + spock_ver = DEFAULT_SPOCK_17 if maj == 17 else DEFAULT_SPOCK + else: + spock_ver = str(spock_ver) # ← force to string + + + # 0.5) Normalize two-digit shorthand (e.g. "50" → "5.0.0") + m = re.fullmatch(r'(\d)(\d)$', spock_ver) + if m: + spock_ver = f"{int(m.group(1))}.{int(m.group(2))}.0" + + # 1) Parse Spock version (abort on bad format) + try: + spv = Version(spock_ver) + except ValueError: + exit_message(f"Invalid Spock version '{spock_ver}'. Aborting.", 1, isJSON) + + # 2) If Spock < 5 ⇒ compatible with any PG + if spv.major < 5: + return + + # — New block: handle pg_ver with “-1” or “-2” suffix + rev = None + rev_match = re.fullmatch(r'(\d+)\.(\d+)-(1|2)$', pg_ver) + if rev_match: + pg_major = int(rev_match.group(1)) + pg_patch = int(rev_match.group(2)) + rev = int(rev_match.group(3)) + + # reject revision “-1” on Spock ≥5 + if rev == 1: + exit_message( + f"Error: PostgreSQL {pg_major}.{pg_patch}-1 is not supported with Spock {spv}; " + "please use the “-2” revision instead.", + 1, + isJSON + ) + # for “-2”, we strip suffix and proceed with pg_major/pg_patch below + # end new block + + # 3) Spock ≥ 5 ⇒ enforce minimum‐patch for each PG major + minimum_patches = { + 15: 13, + 16: 9, + 17: 5, + } + + # 4) Extract PG major and patch (if not already set by rev_match) + if rev_match: + # pg_major, pg_patch are already set + pass + elif "." not in pg_ver: + # bare-major → use its minimum patch + try: + pg_major = int(pg_ver) + except ValueError: + exit_message(f"Invalid PostgreSQL version '{pg_ver}'. Aborting.", 1, isJSON) + if pg_major not in minimum_patches: + allowed = ", ".join(str(m) for m in minimum_patches) + exit_message( + f"Error: Spock {spv} supports only PostgreSQL majors {allowed}; " + f"you have {pg_major}. Aborting.", + 1, + isJSON + ) + pg_patch = minimum_patches[pg_major] + else: + parts = pg_ver.split(".", 2) + if len(parts) < 2: + exit_message(f"Invalid PostgreSQL version '{pg_ver}'. Aborting.", 1, isJSON) + try: + pg_major = int(parts[0]) + pg_patch = int(parts[1]) + except ValueError: + exit_message(f"Invalid PostgreSQL version '{pg_ver}'. Aborting.", 1, isJSON) + + # 5) Major must be supported + if pg_major not in minimum_patches: + allowed = ", ".join(str(m) for m in minimum_patches) + exit_message( + f"Error: Spock {spv} supports only PostgreSQL majors {allowed}; " + f"you have {pg_major}. Aborting.", + 1, + isJSON + ) + + # 6) Enforce minimum‐patch + required = minimum_patches[pg_major] + if pg_patch < required: + exit_message( + f"Error: Spock {spv} requires PostgreSQL {pg_major}.{required} or newer; " + f"you have {pg_major}.{pg_patch}. Aborting.", + 1, + isJSON + ) def get_cpu_info(): try: import cpuinfo diff --git a/devel/setup/compose/README.md b/devel/setup/compose/README.md index b21b2e0a..41ae6c30 100644 --- a/devel/setup/compose/README.md +++ b/devel/setup/compose/README.md @@ -63,9 +63,9 @@ If a local package does not exist, it will fallback to a package available in th The chosen repo corresponds to the URL that you choose to configure: -- http://repo:8000/download corresponds with https://pgedge-download.s3.amazonaws.com/REPO -- http://repo:8000/upstream corresponds with https://pgedge-upstream.s3.amazonaws.com/REPO -- http://repo:8000/devel corresponds with https://pgedge-devel.s3.amazonaws.com/REPO +- http://repo:8000/download corresponds with https://downloads.pgedge.com/platform/repos/download +- http://repo:8000/upstream corresponds with https://downloads.pgedge.com/platform/repos/upstream +- http://repo:8000/devel corresponds with https://downloads.pgedge.com/platform/repos/devel The `out` directory within the build container is mounted in the repo container to enable this setup. diff --git a/devel/setup/compose/proxy_server.py b/devel/setup/compose/proxy_server.py index 8a7de274..97b9b858 100644 --- a/devel/setup/compose/proxy_server.py +++ b/devel/setup/compose/proxy_server.py @@ -33,7 +33,7 @@ def do_GET(self): def proxy_request(self, repo, path): """Fetch content from the upstream server and send it to the client.""" - upstream_url = f"https://pgedge-{repo}.s3.amazonaws.com/REPO/{path}" + upstream_url = f"https://downloads.pgedge.com/platform/repos/{repo}/{path}" self.log_message("Proxying request to %s", upstream_url) try: with urllib.request.urlopen(upstream_url) as response: diff --git a/docs/cli_functions.md b/docs/cli_functions.md index 99162704..f7f5abdc 100644 --- a/docs/cli_functions.md +++ b/docs/cli_functions.md @@ -96,7 +96,6 @@ Use commands in this section to invoke spock extension functionality with the CL |---------|-------------| | [spock node-create](functions/spock-node-create.md) | Define a node for spock. | | [spock node-drop](functions/spock-node-drop.md) | Remove a spock node. | -| [spock node-alter-location](functions/spock-node-alter-location.md) | Set location details for spock node. | | [spock node-list](functions/spock-node-list.md) | Display node table. | | [spock node-add-interface](functions/spock-node-add-interface.md) | Add a new node interface. | | [spock node-drop-interface](functions/spock-node-drop-interface.md) | Delete a node interface. | diff --git a/docs/functions/spock-node-alter-location.md b/docs/functions/spock-node-alter-location.md deleted file mode 100644 index 9d39297b..00000000 --- a/docs/functions/spock-node-alter-location.md +++ /dev/null @@ -1,11 +0,0 @@ - -## SYNOPSIS - ./pgedge spock node-alter-location NODE_NAME LOCATION DB - -## DESCRIPTION - Set location details for spock node. - -## POSITIONAL ARGUMENTS - NODE_NAME - LOCATION - DB diff --git a/docs/functions/spock.md b/docs/functions/spock.md index d1f26fdc..5ed3198c 100644 --- a/docs/functions/spock.md +++ b/docs/functions/spock.md @@ -6,7 +6,6 @@ COMMAND is one of the following: node-create # Define a node for spock. node-drop # Remove a spock node. - node-alter-location # Set location details for spock node. node-list # Display node table. node-add-interface # Add a new node interface. node-drop-interface # Delete a node interface. diff --git a/env.sh b/env.sh index 07a0eb0a..578e51ef 100755 --- a/env.sh +++ b/env.sh @@ -1,12 +1,14 @@ -hubV=25.1.0 -hubVV=25.1.0 +hubV=25.2.0 +hubVV=25.2.0 aceV=$hubV kirkV=$hubV bundle=pgedge api=pgedge -ctlibsV=1.6 +ctlibsV=1.7 + +spock60V=6.0.0-devel-1 spock50V=5.0.0-1 @@ -35,7 +37,7 @@ vectorV=0.8.0-1 bouncerV=1.23.1-1 catV=1.2.0 prompgexpV=0.15.0 -backrestV=2.53.1-1 +backrestV=2.56.0-1 wal2jV=2.6.0-1 citusV=12.1.5-1 diff --git a/requirements.txt b/requirements.txt index 604ab605..0c521bfc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,41 +1,40 @@ ## Core CLI ######### -requests==2.32.2 -paramiko==3.5.0 -psycopg==3.2.3; platform_system=="Linux" -psycopg-binary==3.2.3; platform_system=="Linux" -psutil==6.1.0 +requests==2.32.4 +paramiko==3.5.1 +psycopg==3.2.9; platform_system=="Linux" +psycopg-binary==3.2.9; platform_system=="Linux" +psutil==6.1.1 pypsutil==0.2.0 semantic_version==2.10.0 -six==1.16.0 +six==1.17.0 termcolor==2.5.0 -typing_extensions==4.12.2 -click==8.1.7 +typing_extensions==4.14.1 tabulate==0.9.0 -python-crontab==3.2.0 -certifi==2024.7.4 -charset_normalizer==3.3.2 +python-crontab==3.3.0 +certifi==2024.12.14 +charset_normalizer==3.4.2 gpustat==1.1.1 py-cpuinfo==9.0.0 -prettytable==3.12.0 -Flask==3.0.3 -minio==7.2.10 -tqdm==4.67.0 +prettytable==3.16.0 +Flask==3.1.1 +minio==7.2.15 +tqdm==4.67.1 ## ACE Advanced ##### ordered-set==4.1.0 mpire[dashboard]==2.10.2 rich==13.9.4 -apscheduler==3.10.4 -pandas==2.2.3 +apscheduler==3.11.0 +pandas==2.3.1 pyopenssl==24.3.0 -packaging==23.1 +packaging==23.2 ## pgEdge-HA ######## cdiff==1.0 -urllib3==2.2.2 -PyYAML==6.0.1 +urllib3==2.5.0 +PyYAML==6.0.2 python-etcd==0.4.5 python-dateutil==2.9.0 -cryptography==44.0.1 -ydiff==1.3 +cryptography==44.0.3 +ydiff==1.4.2 diff --git a/src/backrest/backrest.py b/src/backrest/backrest.py index 2a805509..997742fe 100755 --- a/src/backrest/backrest.py +++ b/src/backrest/backrest.py @@ -30,8 +30,7 @@ def fetch_config(): "repo1-retention-full-type", "repo1-path", "repo1-host-user", "repo1-host", "repo1-cipher-type", "log-level-console", "repo1-type", "process-max", "compress-level", "pg1-path", - "pg1-user", "pg1-database", "db-socket-path", "pg1-port", - "pg1-host" + "pg1-user", "pg1-database", "db-socket-path", "pg1-port" ] for param in params: config[param] = util.get_value("BACKUP", param) diff --git a/src/backrest/install-backrest.py b/src/backrest/install-backrest.py index 461168d1..e9ffef32 100644 --- a/src/backrest/install-backrest.py +++ b/src/backrest/install-backrest.py @@ -58,7 +58,6 @@ def configure_backup_settings(): "pg1-path": "xx", "pg1-user": "xx", "pg1-port": "5432", - "pg1-host": "127.0.0.1", "db-socket-path": "/tmp", "global:archive-push": { "compress-level": "3" diff --git a/src/conf/versions.sql b/src/conf/versions.sql index c6218575..b7eb42c8 100644 --- a/src/conf/versions.sql +++ b/src/conf/versions.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS hub; CREATE TABLE hub(v TEXT NOT NULL PRIMARY KEY, c TEXT NOT NULL, d TEXT NOT NULL); -INSERT INTO hub VALUES ('25.1.0', '', '20250626'); +INSERT INTO hub VALUES ('25.2.0', '', '20250815'); DROP VIEW IF EXISTS v_versions; DROP VIEW IF EXISTS v_products; @@ -74,17 +74,17 @@ CREATE TABLE extensions ( preload_name TEXT NOT NULL, default_conf TEXT NOT NULL ); -INSERT INTO extensions VALUES ('spock33', 'spock', 1, 'spock', +INSERT INTO extensions VALUES ('spock40', 'spock', 1, 'spock', 'wal_level=logical | max_worker_processes=12 | max_replication_slots=16 | max_wal_senders=16 | hot_standby_feedback=on | wal_sender_timeout=5s | track_commit_timestamp=on | spock.conflict_resolution=last_update_wins | spock.save_resolutions=on | spock.conflict_log_level=DEBUG'); -INSERT INTO extensions VALUES ('spock40', 'spock', 1, 'spock', +INSERT INTO extensions VALUES ('spock50', 'spock', 1, 'spock', 'wal_level=logical | max_worker_processes=12 | max_replication_slots=16 | max_wal_senders=16 | hot_standby_feedback=on | wal_sender_timeout=5s | track_commit_timestamp=on | spock.conflict_resolution=last_update_wins | spock.save_resolutions=on | spock.conflict_log_level=DEBUG'); -INSERT INTO extensions VALUES ('spock50', 'spock', 1, 'spock', +INSERT INTO extensions VALUES ('spock60', 'spock', 1, 'spock', 'wal_level=logical | max_worker_processes=12 | max_replication_slots=16 | max_wal_senders=16 | hot_standby_feedback=on | wal_sender_timeout=5s | track_commit_timestamp=on | spock.conflict_resolution=last_update_wins | @@ -139,6 +139,7 @@ INSERT INTO projects VALUES ('hub', 'app', 0, 0, 'hub', 0, 'https://github.com/p INSERT INTO releases VALUES ('hub', 1, 'hub', '', '', 'hidden', '', 1, '', '', ''); INSERT INTO versions VALUES ('hub', (select v from hub), '', 1, (select d from hub), '', '', ''); +INSERT INTO versions VALUES ('hub', '25.1.0', '', 0, '20250626', '', '', ''); INSERT INTO versions VALUES ('hub', '25.0.0', '', 0, '20250603', '', '', ''); INSERT INTO versions VALUES ('hub', '24.10.13', '', 0, '20250509', '', '', ''); INSERT INTO versions VALUES ('hub', '24.10.11', '', 0, '20250224', '', '', ''); @@ -320,17 +321,6 @@ INSERT INTO versions VALUES ('snowflake-pg17', '2.2-1', 'amd, arm', 1, '20240626 -- ## SPOCK (parent project) ############ INSERT INTO projects VALUES ('spock', 'pge', 4, 0, '', 1, 'https://github.com/pgedge/spock/tags', 'spock', 1, 'spock.png', 'Logical Rep w/ Conflict Resolution', 'https://github.com/pgedge/spock/', 'pg_spock, pgsspock, vulcan'); - --- ## SPOCK33 ########################### -INSERT INTO releases VALUES ('spock33-pg15', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); -INSERT INTO releases VALUES ('spock33-pg16', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); - -INSERT INTO versions VALUES ('spock33-pg15', '3.3.6-1', 'amd, arm', 1, '20240820', 'pg15', '', ''); -INSERT INTO versions VALUES ('spock33-pg16', '3.3.6-1', 'amd, arm', 1, '20240820', 'pg16', '', ''); - -INSERT INTO versions VALUES ('spock33-pg15', '3.3.5-1', 'amd, arm', 0, '20240607', 'pg15', '', ''); -INSERT INTO versions VALUES ('spock33-pg16', '3.3.5-1', 'amd, arm', 0, '20240607', 'pg16', '', ''); - -- ## SPOCK40 ########################### INSERT INTO releases VALUES ('spock40-pg15', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); INSERT INTO releases VALUES ('spock40-pg16', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); @@ -350,14 +340,23 @@ INSERT INTO versions VALUES ('spock40-pg17', '4.0.8-1', 'amd, arm', 0, '20241218 -- ## spock50 ########################### -INSERT INTO releases VALUES ('spock50-pg15', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); -INSERT INTO releases VALUES ('spock50-pg16', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); -INSERT INTO releases VALUES ('spock50-pg17', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); +INSERT INTO releases VALUES ('spock50-pg15', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); +INSERT INTO releases VALUES ('spock50-pg16', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); +INSERT INTO releases VALUES ('spock50-pg17', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); INSERT INTO versions VALUES ('spock50-pg15', '5.0.0-1', 'amd, arm', 1, '20250715', 'pg15', '', ''); INSERT INTO versions VALUES ('spock50-pg16', '5.0.0-1', 'amd, arm', 1, '20250715', 'pg16', '', ''); INSERT INTO versions VALUES ('spock50-pg17', '5.0.0-1', 'amd, arm', 1, '20250715', 'pg17', '', ''); +-- ## spock60 ########################### +INSERT INTO releases VALUES ('spock60-pg15', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); +INSERT INTO releases VALUES ('spock60-pg16', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); +INSERT INTO releases VALUES ('spock60-pg17', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); + +INSERT INTO versions VALUES ('spock60-pg15', '6.0.0-devel-1', 'amd, arm', 1, '20250801', 'pg15', '', ''); +INSERT INTO versions VALUES ('spock60-pg16', '6.0.0-devel-1', 'amd, arm', 1, '20250801', 'pg16', '', ''); +INSERT INTO versions VALUES ('spock60-pg17', '6.0.0-devel-1', 'amd, arm', 1, '20250801', 'pg17', '', ''); + -- ## LOLOR ############################# INSERT INTO projects VALUES ('lolor', 'pge', 4, 0, '', 1, 'https://github.com/pgedge/lolor/tags', 'spock', 1, 'spock.png', 'Logical Replication of Large Objects', 'https://github.com/pgedge/lolor/#spock', 'lola, lolah, kinks'); @@ -421,7 +420,7 @@ INSERT INTO versions VALUES ('hypopg-pg16', '1.4.1-1', 'amd, arm', 1, '20230509 INSERT INTO projects VALUES ('ctlibs', 'pge', 0, 0, '', 3, 'https://github.com/pgedge/cli', 'ctlibs', 0, 'ctlibs.png', 'ctlibs', 'https://github.com/pgedge/cli', ''); INSERT INTO releases VALUES ('ctlibs', 2, 'ctlibs', 'pgEdge Libs', '', 'prod', '', 1, '', '', ''); -INSERT INTO versions VALUES ('ctlibs', '1.6', '', 1, '20240925', '', '', ''); +INSERT INTO versions VALUES ('ctlibs', '1.7', '', 1, '20250729', '', '', ''); -- ## PGCAT ############################# INSERT INTO projects VALUES ('pgcat', 'pge', 11, 5433, '', 3, 'https://github.com/pgedge/pgcat/tags', @@ -440,8 +439,8 @@ INSERT INTO projects VALUES ('backrest', 'pge', 11, 0, '', 3, 'http://pgbackrest 'backrest', 0, 'backrest.png', 'Backup & Restore', 'http://pgbackrest.org', 'pg_backrest, pgbackrest'); INSERT INTO releases VALUES ('backrest', 2, 'backrest', 'pgBackRest', '', 'test', '', 1, 'MIT', 'EL', ''); -INSERT INTO versions VALUES ('backrest', '2.53.1-1', 'amd, arm', 1, '20240912', '', '', ''); -INSERT INTO versions VALUES ('backrest', '2.53-1', 'amd, arm', 0, '20240729', '', '', ''); +INSERT INTO versions VALUES ('backrest', '2.56.0-1', 'amd, arm', 1, '20240805', '', '', ''); +INSERT INTO versions VALUES ('backrest', '2.53.1-1', 'amd, arm', 0, '20240912', '', '', ''); -- ## PATRONI ########################### INSERT INTO projects VALUES ('patroni', 'app', 11, 0, '', 4, 'https://github.com/pgedge/pgedge-patroni/release', diff --git a/src/pgXX/init-pgXX.py b/src/pgXX/init-pgXX.py index 6111c7db..9c983d40 100644 --- a/src/pgXX/init-pgXX.py +++ b/src/pgXX/init-pgXX.py @@ -5,6 +5,7 @@ import util, startup import argparse, os, sys, shutil, subprocess, getpass, json +import shlex MY_HOME = os.getenv("MY_HOME", "") @@ -196,10 +197,14 @@ def fatal_error(p_msg): util.update_postgresql_conf(pgver, i_port) if util.get_platform() == "Linux": - os.system("cp " + pgver + "/genSelfCert.sh " + pg_data + "/.") - os.system(pg_data + "/genSelfCert.sh") - -os.system("cp " + pgver + "/pg_hba.conf.nix " + pg_data + "/pg_hba.conf") + gen_cert_src = os.path.join(pgver, "genSelfCert.sh") + gen_cert_dst = os.path.join(pg_data, "genSelfCert.sh") + os.system(f'cp {shlex.quote(gen_cert_src)} {shlex.quote(gen_cert_dst)}') + os.system(f'{shlex.quote(gen_cert_dst)}') + +pg_hba_src = os.path.join(pgver, "pg_hba.conf.nix") +pg_hba_dst = os.path.join(pg_data, "pg_hba.conf") +os.system(f'cp {shlex.quote(pg_hba_src)} {shlex.quote(pg_hba_dst)}') if is_password: pg_pass_file = util.remember_pgpassword(pg_password, "*", "*", "*", os_user) diff --git a/src/pgXX/remove-pgXX.py b/src/pgXX/remove-pgXX.py index 28203cff..fcd77ce9 100644 --- a/src/pgXX/remove-pgXX.py +++ b/src/pgXX/remove-pgXX.py @@ -15,5 +15,6 @@ isRM_DATA = os.getenv("isRM_DATA", "False") if isRM_DATA == "True": util.message("Removing 'data' directories at your request") - util.echo_cmd(f"sudo rm -r data/{pgver}") + data_dir = util.get_column("datadir", pgver) + util.echo_cmd(f"sudo rm -r {data_dir}") util.echo_cmd(f"sudo rm -r data/logs/{pgver}")