From d3ffb8b022fcf82c2e5280f95ea197a0c9c1cde4 Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Thu, 5 Jun 2025 18:30:29 +0500 Subject: [PATCH 01/42] Updates default repo URL in install.py (#325) (#326) Co-authored-by: hayee-bhatti --- cli/scripts/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/scripts/install.py b/cli/scripts/install.py index f7927eea..c8094ee4 100644 --- a/cli/scripts/install.py +++ b/cli/scripts/install.py @@ -4,7 +4,7 @@ import sys, os, tarfile, platform VER = "25.0.0" -REPO = os.getenv("REPO", "https://pgedge-upstream.s3.amazonaws.com/REPO") +REPO = os.getenv("REPO", "https://pgedge-download.s3.amazonaws.com/REPO") if sys.version_info < (3, 9): maj = sys.version_info.major From 3c9bcc518b02e08b994672423ea18cb62abca609 Mon Sep 17 00:00:00 2001 From: hayee-bhatti Date: Thu, 12 Jun 2025 17:02:31 +0500 Subject: [PATCH 02/42] Enhance build_to_devel.sh to support spock daily builds alongside stable platform builds - Introduce two build modes, stable and current - "stable" builds produce a platform build with all stable components with the latest CLI - "current" mode additionally produces spock daily builds (from in-development main or specified branch) - Publish each build to its appropriate S3 location within the pgedge-devel repo - Adjust S3 lifecycle policies dynamically based on the build mode Corresponding stable/current workflows in the CLI project now call this script with the appropriate mode and switches --- devel/util/build_to_devel.sh | 130 ++++++++++++++++++++++++++--------- 1 file changed, 96 insertions(+), 34 deletions(-) diff --git a/devel/util/build_to_devel.sh b/devel/util/build_to_devel.sh index eed0ca53..b40f68e1 100755 --- a/devel/util/build_to_devel.sh +++ b/devel/util/build_to_devel.sh @@ -1,6 +1,79 @@ #!/bin/bash cd "$(dirname "$0")" +# Enhanced build script with support for 'stable' and 'current' build modes. +# - 'stable' mode: Builds packages with all stable components + the latest CLI + offline bundle +# suitable for reliable, production-ready releases. +# Stable builds are pushed to pgedge-devel repo into the REPO/stable/ +# +# - 'current' mode: Extends 'stable' mode by including the latest (in-development/unstable) spock +# (e.g. spock50) which will be extended in future to contain more of the in dev/unstable components +# Current builds are pushed to pgedge-devel repo into the REPO/current/ +# +# This script is now the primary build tool, replacing the older build_to_devel.sh, +# and is called by corresponding workflows in pgedge/cli for stable (amd8 and arm9) and current (amd8 and arm9) +# builds. It supports both automated daily runs and manual triggers with customizable options via switches. +# +# New switches introduced: +# - -m MODE: Specifies the build mode ('stable' or 'current', defaults to 'stable'). +# - -c COMPONENT: Required for 'current' mode, specifies the Spock component (e.g., spock50). +# - -b BRANCH: For 'current' mode, sets the branch for the Spock component (defaults to 'main'). +# - -p PGVERS: For 'current' mode, comma-separated PG versions to build (defaults to '15,16,17'). +# - -d S3SUBDIR: Sets the S3 subdirectory for uploads (defaults to MMDD, e.g., 0612). +# - -x: Enables cleaning the S3 subdirectory before upload. +# +# The script handles S3 uploads, lifecycle policies, and build artifacts, adapting behavior +# based on the selected mode. +# +# +# Note : 'stable' mode relies solely on -d and -x +# while additional inputs -c, -b -p are associated with 'current' mode only + +#Argument parsing using getopts for stable/current modes and options +MODE="stable" +COMPONENTNAME="" +BRANCH="main" +PGVERS="15,16,17" +subdir=$(date +%m%d) +cleaner="" + +show_help() { + cat << EOF +Usage: $0 [OPTIONS] + -m MODE Build mode: 'stable' (default) or 'current' (includes in-dev spock) + -c COMPONENT Spock component to build (required in current mode, e.g. spock50) + -b BRANCH Branch to use for spock component (default: main, current mode only) + -p PGVERS Comma-separated PG versions (default: 15,16,17, current mode only) + -d S3SUBDIR S3 subdirectory (default: MMDD) + -x Clean S3 subdir before upload, optional + -h Show help + +Examples: + $0 -d 0522 -x + $0 -m stable -d 0522 + $0 -m current -c spock50 -b main -p 15,16,17 -d 0522 +EOF +} + +while getopts "m:c:b:p:d:xh" opt; do + case $opt in + m) MODE="$OPTARG" ;; + c) COMPONENTNAME="$OPTARG" ;; + b) BRANCH="$OPTARG" ;; + p) PGVERS="$OPTARG" ;; + d) subdir="$OPTARG" ;; + x) cleaner="--clean" ;; + h) show_help; exit 0 ;; + \?) show_help; exit 1 ;; + esac +done + +if [[ "$MODE" == "current" && -z "$COMPONENTNAME" ]]; then + echo "[ERROR] -c COMPONENT is required in current mode" + show_help + exit 1 +fi + source $PGE/env.sh # Show progress updates if stdout is a terminal; @@ -77,41 +150,27 @@ check_s3_access() { } # --- Pre-checks (Step -1) --- -step -1 "Pre-checks: Environment, Disk Space & S3 Access" +step -2 "Pre-checks: Environment, Disk Space & S3 Access" check_env_vars check_disk_space check_s3_access -# ------------------------------- -# New Argument Parsing for Subdirectory and Clean Flag -# Usage: ./build_to_devel.sh [subdirectory] [--clean] -# If no custom subdirectory is provided then MMDD is used and --clean is not set. -subdir=$(date +%m%d) -cleaner="" - -# Parse input arguments -if [ "$1" == "--clean" ]; then - cleaner="--clean" - shift -fi - -if [ -n "$1" ]; then - subdir="$1" - if [ "$2" == "--clean" ]; then - cleaner="--clean" +# Step -1: If mode is current, set the s3 prefix and execute get-and-build-spock.sh that fetches latest source from spock +# branch and builds its binaries using the pgbin-build project. +# Else, in stable mode, only set the s3 prefix. +if [[ "$MODE" == "current" ]]; then + step -1 "set current S3 prefix, run get-and-build-spock.sh to build latest spock #####################" + prefix="REPO/current/$subdir" + if ! "$BLD/get-and-build-spock.sh" -b "$BRANCH" -c "$COMPONENTNAME" -p "$PGVERS"; then + echo "[ERROR] get-and-build-spock.sh failed!" + exit 1 fi +else + step -1 "set stable S3 prefix #########################" + prefix="REPO/stable/$subdir" fi - -echo "# Using subdirectory: $subdir" -if [ "$cleaner" == "--clean" ]; then - echo "# Clean flag is set" -fi - -# Calculate S3 path prefix (REPO/stable/) -prefix="REPO/stable/$subdir" echo "# Using S3 prefix: $prefix" -# ------------------------------- step 0 "## initialization #######################" echo "# BUCKET = $BUCKET" @@ -143,7 +202,8 @@ cmd "git pull" step 3a "building tgz bundle ####################" echo "Building tgz bundle : $offline_tgz_bndl" -./make_tgz.sh +# call make_tgz.sh which also now takes the mode (stable/current) +./make_tgz.sh -m "$MODE" sleep 9 # Verify that the offline tarball was created if [ ! -f "$OUT/$offline_tgz_bndl" ]; then @@ -172,13 +232,16 @@ cmd "aws --region $REGION s3 cp . $BUCKET/$prefix $flags $PROGRESS_FLAG" step 6a "recopy offline repo tgz to S3 with headers ############################" cmd "aws --region $REGION s3 cp $offline_tgz_bndl $BUCKET/$prefix/ --acl public-read --content-disposition \"attachment; filename=$offline_tgz_bndl\" $PROGRESS_FLAG" -# Define a lifecycle policy JSON for all objects under the REPO/stable to auto expire after 7 days -step 6b "Set lifecycle policy for the objects under REPO/stable to auto expire/delete after 7 days" +# Compute the base prefix (e.g., REPO/stable/ or REPO/current/) +base_prefix="${prefix%/*}/" + +# Define a lifecycle policy JSON dynamically for all objects under the base prefix +step 6b "Set lifecycle policy for objects under $base_prefix to auto expire/delete after 7 days" policy='{ "Rules": [ { - "ID": "ExpireStableBuilds", - "Filter": { "Prefix": "REPO/stable/" }, + "ID": "ExpireBuilds_'"$MODE"'", + "Filter": { "Prefix": "'"$base_prefix"'" }, "Status": "Enabled", "Expiration": { "Days": 7 } } @@ -188,6 +251,5 @@ cmd "aws --region $REGION s3api put-bucket-lifecycle-configuration --bucket $BUC step 7 "Goodbye! ##############################" echo "Script completed successfully" -exit 0 - +exit 0 From f7ac95abab41e8fadc7272f629781632de9ef8a0 Mon Sep 17 00:00:00 2001 From: hayee-bhatti Date: Thu, 12 Jun 2025 17:37:08 +0500 Subject: [PATCH 03/42] Enhance env.sh and make_tgz.sh to integrate with recent build_to_devel.sh changes for stable/current modes handling - Update env.sh to define removeComponentFromOut (that is used in make_tgz.sh) for stripping the Spock (unstable) component in stable mode builds - Enhance make_tgz.sh to support stable/current modes - Align with build_to_devel.sh for a consistent mode based build process --- env.sh | 4 ++++ make_tgz.sh | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/env.sh b/env.sh index 919e9ee1..bd119765 100755 --- a/env.sh +++ b/env.sh @@ -14,6 +14,10 @@ spock40V=4.0.10-1 spock33V=3.3.6-1 +# removeComponentFromOut: Specifies the Spock component (e.g., spock50) to exclude from stable mode builds in make_tgz.sh. +# This variable is ignored in current mode builds, which include all components. +removeComponentFromOut=spock50 + lolorV=1.2-1 snwflkV=2.2-1 diff --git a/make_tgz.sh b/make_tgz.sh index 38a8ec7e..00889ff8 100755 --- a/make_tgz.sh +++ b/make_tgz.sh @@ -1,6 +1,11 @@ #!/bin/bash cd "$(dirname "$0")" +# Script to generate pgedge builds and the corresponding tgz bundles with mode-based customization. +# Supports '-m MODE' switch for 'stable' (default) or 'current' modes. +# In 'stable' mode, after generating builds, strips specified unstable components using 'removeComponentFromOut' from env.sh. +# In 'current' mode, includes all components. + TGZ_REPO="https://pgedge-download.s3.amazonaws.com/REPO" source env.sh @@ -18,6 +23,33 @@ cmd () { } +show_help() { + cat << EOF +Usage: $0 [OPTIONS] + -m MODE Build mode: 'stable' (default) or 'current' + -h Show help + +Examples: + $0 + $0 -m current +EOF +} + +MODE="stable" +while getopts "m:h" opt; do + case $opt in + m) MODE="$OPTARG" ;; + h) show_help; exit 0 ;; + \?) show_help; exit 1 ;; + esac +done +# check that the mode passed is either stable or current +if [[ "$MODE" != "stable" && "$MODE" != "current" ]]; then + echo "[ERROR] Invalid mode: $MODE" + show_help + exit 1 +fi + ## MAINLINE ################################### vers="15 16 17" @@ -27,6 +59,23 @@ for ver in ${vers}; do cmd "./build_all.sh $ver" done +sleep 5 +# If mode is 'stable' and removeComponentFromOut has a value defined in env.sh, +# strip the corresponding unstable component packages (e.g., spock50) from the build. +# This ensures only stable components are included in the tgz bundle. +# In 'current' mode, this step is skipped, preserving all stable and unstable components +# Wildcards (*) in removeComponentFromOut are not supported. +removeComponentFromOut="$(echo -n "$removeComponentFromOut" | xargs)" +if [[ "$MODE" == "stable" && -n "$removeComponentFromOut" ]]; then + if [[ "$removeComponentFromOut" == *"*"* ]]; then + echo "[ERROR] removeComponentFromOut should not contain a wildcard (*). Skipping removal." + else + echo "[INFO] [STABLE MODE] Attempting to remove files matching: $OUT/${removeComponentFromOut}*.tgz" + removed_files=$(rm -vf "$OUT/${removeComponentFromOut}"*.* 2>&1 | wc -l) + echo "[INFO] Removed $removed_files files for $removeComponentFromOut." + fi +fi + ./bp.sh # remove large ctlib tarballs of different architecture From 65f5bae78417038b9ddcadb4d151139f70295afe Mon Sep 17 00:00:00 2001 From: hayee-bhatti Date: Thu, 12 Jun 2025 22:15:25 +0500 Subject: [PATCH 04/42] Renaming/updating the existing daily builds workflows for amd/arm - Naming them appropriately - Making necessary adjustments (switches) to account for the changes in build scripts. --- ...d-devel-amd8.yml => stable-amd8-daily-build-devel.yml} | 6 +++--- ...d-devel-arm9.yml => stable-arm9-daily-build-devel.yml} | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) rename .github/workflows/{daily-build-devel-amd8.yml => stable-amd8-daily-build-devel.yml} (96%) rename .github/workflows/{daily-build-devel-arm9.yml => stable-arm9-daily-build-devel.yml} (96%) diff --git a/.github/workflows/daily-build-devel-amd8.yml b/.github/workflows/stable-amd8-daily-build-devel.yml similarity index 96% rename from .github/workflows/daily-build-devel-amd8.yml rename to .github/workflows/stable-amd8-daily-build-devel.yml index 0dc60bd8..aaa27e56 100644 --- a/.github/workflows/daily-build-devel-amd8.yml +++ b/.github/workflows/stable-amd8-daily-build-devel.yml @@ -82,7 +82,7 @@ jobs: # Clean flag handling # If true is chosen in the clean dropdown, set the --clean flag if [[ "${{ inputs.clean_prefix }}" == "true" ]]; then - CLEAN_FLAG="--clean" + CLEAN_FLAG="-x" echo "Clean flag is enabled" else CLEAN_FLAG="" @@ -146,8 +146,8 @@ jobs: echo "Launching daily build script..." cd $DEV - echo "Executing: ./build_to_devel.sh $PREFIX $CLEAN_FLAG" - ./build_to_devel.sh "$PREFIX" $CLEAN_FLAG > "$LOGFILE_PATH" 2>&1 + echo "Executing: ./build_to_devel.sh -m stable -d $PREFIX $CLEAN_FLAG" + ./build_to_devel.sh -m stable -d "$PREFIX" $CLEAN_FLAG > "$LOGFILE_PATH" 2>&1 EXIT_CODE=$? echo "Build finished with exit code: $EXIT_CODE" diff --git a/.github/workflows/daily-build-devel-arm9.yml b/.github/workflows/stable-arm9-daily-build-devel.yml similarity index 96% rename from .github/workflows/daily-build-devel-arm9.yml rename to .github/workflows/stable-arm9-daily-build-devel.yml index 5e54be69..1d224fbb 100644 --- a/.github/workflows/daily-build-devel-arm9.yml +++ b/.github/workflows/stable-arm9-daily-build-devel.yml @@ -80,9 +80,9 @@ jobs: fi # Clean flag handling - # If true is chosen in the clean dropdown, set the --clean flag + # If true is chosen in the clean dropdown, set the -x flag if [[ "${{ inputs.clean_prefix }}" == "true" ]]; then - CLEAN_FLAG="--clean" + CLEAN_FLAG="-x" echo "Clean flag is enabled" else CLEAN_FLAG="" @@ -146,8 +146,8 @@ jobs: echo "Launching daily build script..." cd $DEV - echo "Executing: ./build_to_devel.sh $PREFIX $CLEAN_FLAG" - ./build_to_devel.sh "$PREFIX" $CLEAN_FLAG > "$LOGFILE_PATH" 2>&1 + echo "Executing: ./build_to_devel.sh -m stable -d $PREFIX $CLEAN_FLAG" + ./build_to_devel.sh -m stable -d "$PREFIX" $CLEAN_FLAG > "$LOGFILE_PATH" 2>&1 EXIT_CODE=$? echo "Build finished with exit code: $EXIT_CODE" From 89071a6d1a436003352a21b886a5f6e4fecae295 Mon Sep 17 00:00:00 2001 From: hayee-bhatti Date: Thu, 12 Jun 2025 23:46:21 +0500 Subject: [PATCH 05/42] Add current-mode workflows for pgEdge platform to produce in-dev Spock builds along with platform packages - Introduces AMD8/ARM9 workflows for daily 'current' builds containing: * Stable platform components + latest Spock (e.g. spock50) * Default from main branch but configurable in manual mode - Supports both scheduled (defaults) and manual (custom) triggers - Publishes to pgedge-devel in REPO/current/ with 7-day retention - Captures build logs and Spock binaries/sources as artifacts --- .../current-amd8-daily-build-devel.yml | 271 ++++++++++++++++++ .../current-arm9-daily-build-devel.yml | 271 ++++++++++++++++++ 2 files changed, 542 insertions(+) create mode 100644 .github/workflows/current-amd8-daily-build-devel.yml create mode 100644 .github/workflows/current-arm9-daily-build-devel.yml diff --git a/.github/workflows/current-amd8-daily-build-devel.yml b/.github/workflows/current-amd8-daily-build-devel.yml new file mode 100644 index 00000000..c1834e1d --- /dev/null +++ b/.github/workflows/current-amd8-daily-build-devel.yml @@ -0,0 +1,271 @@ +name: Current Daily Build Devel - amd8 + +# Define default environment variables for carrying defaults at the top for easy modification +env: + DEFAULT_CLI_BRANCH: "REL25_01" # Default CLI branch for scheduled runs + DEFAULT_MODE: "current" # Always "current" for this workflow + DEFAULT_COMPONENT: "spock50" # Default spock component name + DEFAULT_BRANCH: "main" # Default branch for the spock component + DEFAULT_CLEAN_FLAG: "false" # Default clean flag for scheduled runs + +# Triggers: Scheduled at 12:00 AM UTC and manual workflow dispatch +on: + workflow_dispatch: + inputs: + cli_branch: + description: "Select the CLI branch to build from (e.g. REL25_01)" + required: true + default: "REL25_01" + type: choice + options: + - REL25_01 + - REL24_10 + + component: + description: "Spock in-dev component to additionally build (e.g. spock50)" + required: true + default: "spock50" + type: string + + branch: + description: "Branch to use for spock component (e.g. main)" + required: true + default: "main" + type: string + + prefix_selector: + description: "Choose S3 subdirectory strategy (default: MMDD, or custom)" + required: true + default: "default" + type: choice + options: + - default + - custom + + custom_prefix: + description: "If 'custom' selected above, provide sub-directory name" + required: false + type: string + + clean_prefix: + description: "Clean the repo sub-directory before copying packages" + required: false + type: choice + options: + - "false" + - "true" + default: "false" + + schedule: + - cron: "0 0 * * *" # 12:00 AM UTC, 8pm EST.. 1 hour after stable build at 11:00 PM UTC + +jobs: + build-current-amd8: + runs-on: [self-hosted, amd8] + + steps: + # Resolve parameters (scheduled vs manual) and set up timestamped log directory + - name: Set build parameters and logfile + id: vars + run: | + source ~/.bashrc + + echo "Resolving build parameters for CURRENT mode workflow..." + + TODAY=$(date +%m%d) + RUNNER_NAME="${{ runner.name }}" + + # Create timestamped log directory with mmddhhmmss format + TIMESTAMP=$(date +%m%d%H%M%S) + LOG_DIR="/tmp/current-build-$TIMESTAMP" + mkdir -p "$LOG_DIR" + echo "Created log directory: $LOG_DIR" + + # Determine parameters based on trigger type + if [[ "${{ github.event_name }}" == "schedule" ]]; then + # Scheduled run: use defaults + PREFIX="$TODAY" + CLEAN_FLAG="" + SELECTED_CLI_BRANCH="${DEFAULT_CLI_BRANCH}" + COMPONENT="${DEFAULT_COMPONENT}" + BRANCH="${DEFAULT_BRANCH}" + echo "Scheduled run: Prefix=$PREFIX, Component=$COMPONENT, Branch=$BRANCH, Clean flag not set, CLI Branch=$SELECTED_CLI_BRANCH" + + else + # Manual run: process inputs + if [[ "${{ inputs.prefix_selector }}" == "default" ]]; then + PREFIX="$TODAY" + echo "Manual run: Using default prefix: $PREFIX" + else + # Custom prefix validation + if [[ "${{ inputs.prefix_selector }}" == "custom" && -z "${{ inputs.custom_prefix }}" ]]; then + echo "Error: Custom prefix selected but no input provided." + exit 1 + fi + PREFIX="${{ inputs.custom_prefix }}" + echo "Manual run: Using custom prefix: $PREFIX" + fi + + # Clean flag handling + if [[ "${{ inputs.clean_prefix }}" == "true" ]]; then + CLEAN_FLAG="-x" + echo "Clean flag is enabled" + else + CLEAN_FLAG="" + echo "Clean flag is disabled" + fi + + SELECTED_CLI_BRANCH="${{ inputs.cli_branch }}" + COMPONENT="${{ inputs.component }}" + BRANCH="${{ inputs.branch }}" + echo "Manual run: Component=$COMPONENT, Branch=$BRANCH, CLI Branch=$SELECTED_CLI_BRANCH" + fi + + # Set logfile path, matching stable workflow with "current" label + LOGFILE_NAME="build_to_devel_current-${RUNNER_NAME}-${TIMESTAMP}.log" + LOGFILE_PATH="$LOG_DIR/$LOGFILE_NAME" + + # Export variables + echo "RUNNER_NAME=$RUNNER_NAME" >> $GITHUB_ENV + echo "MODE=${DEFAULT_MODE}" >> $GITHUB_ENV + echo "COMPONENT=$COMPONENT" >> $GITHUB_ENV + echo "BRANCH=$BRANCH" >> $GITHUB_ENV + echo "PREFIX=$PREFIX" >> $GITHUB_ENV + echo "CLEAN_FLAG=$CLEAN_FLAG" >> $GITHUB_ENV + echo "LOG_DIR=$LOG_DIR" >> $GITHUB_ENV + echo "LOGFILE_PATH=$LOGFILE_PATH" >> $GITHUB_ENV + echo "CLI_BRANCH=$SELECTED_CLI_BRANCH" >> $GITHUB_ENV + + echo "Log file will be created at: $LOGFILE_PATH" + + # Display the resolved configuration for debugging + - name: Show effective configuration + run: | + echo "=== CURRENT BUILD WORKFLOW CONFIGURATION ===" + echo "Runner: $RUNNER_NAME" + echo "Mode: $MODE" + echo "Component: $COMPONENT" + echo "Branch: $BRANCH" + echo "S3 Prefix: $PREFIX" + echo "Clean flag: $CLEAN_FLAG" + echo "Log directory: $LOG_DIR" + echo "Log file path: $LOGFILE_PATH" + echo "CLI branch: $CLI_BRANCH" + echo "==============================================" + + # Verify environment details and switch to the specified CLI branch + - name: Confirm environment and switch CLI branch + id: cli_branch_checkout + run: | + source ~/.bashrc + echo "OS info:" + cat /etc/os-release + echo "Architecture:" + arch + + echo "Navigating to CLI directory: $PGE" + cd $PGE + + echo "Current branch and status:" + git branch + git status + + echo "Stashing changes if any..." + git stash push -u -m "Temp stash for current build" || true + + echo "Switching to branch: $CLI_BRANCH" + git checkout "$CLI_BRANCH" || true + echo "Branch checkout completed" + + # Execute the build script and collect artifacts into src/ and bin/ + - name: Run build and push to devel repo + id: build_to_devel + run: | + source ~/.bashrc + + echo "Launching build script in CURRENT mode..." + cd $DEV + + # Construct and run the build command + BUILD_CMD="./build_to_devel_2.sh -m $MODE -c $COMPONENT -b $BRANCH -d $PREFIX" + if [[ -n "$CLEAN_FLAG" ]]; then + BUILD_CMD="$BUILD_CMD $CLEAN_FLAG" + fi + + echo "Executing: $BUILD_CMD" + $BUILD_CMD > "$LOGFILE_PATH" 2>&1 + EXIT_CODE=$? + + echo "Build finished with exit code: $EXIT_CODE" + + # Collect artifacts into src/ and bin/ + echo "Collecting source and binary artifacts..." + mkdir -p "$LOG_DIR/src" "$LOG_DIR/bin" + + echo "Copying source files from $SOURCE/${COMPONENT}*" + if ls "$SOURCE/${COMPONENT}"* 1> /dev/null 2>&1; then + cp "$SOURCE/${COMPONENT}"* "$LOG_DIR/src/" 2>/dev/null || true + echo "Source files copied" + else + echo "No source files found" + fi + + echo "Copying binaries from $IN/postgres/$COMPONENT/${COMPONENT}*" + if ls "$IN/postgres/$COMPONENT/${COMPONENT}"* 1> /dev/null 2>&1; then + cp "$IN/postgres/$COMPONENT/${COMPONENT}"* "$LOG_DIR/bin/" 2>/dev/null || true + echo "Binary files copied" + else + echo "No binary files found" + fi + + # List collected artifacts + echo "=== COLLECTED ARTIFACTS ===" + ls -la "$LOG_DIR/" + ls -la "$LOG_DIR/src/" || echo "No source files" + ls -la "$LOG_DIR/bin/" || echo "No binary files" + echo "==========================" + + if [[ $EXIT_CODE -ne 0 ]]; then + echo "Build failed. Artifacts collected for debugging." + exit $EXIT_CODE + fi + + # Compress the log directory for artifact upload + - name: Compress log directory + if: always() + run: | + echo "Compressing log directory: $LOG_DIR" + cd "$(dirname "$LOG_DIR")" + tar -czf "${LOG_DIR}.tar.gz" "$(basename "$LOG_DIR")" + echo "Compressed archive created: ${LOG_DIR}.tar.gz" + + # Upload the compressed log archive as an artifact + - name: Upload build log artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: current-build-log + path: ${{ env.LOG_DIR }}.tar.gz + if-no-files-found: warn + + # Clean up historical build artifacts and temporary files + - name: Cleanup build artifacts + if: steps.build_to_devel.outcome == 'success' + continue-on-error: true + run: | + source ~/.bashrc + + run_day=$(date +%j) + TARGET="$HIST/devel-$run_day" + echo "Cleaning up historical build directory: $TARGET" + if [ -d "$TARGET" ]; then + rm -rf "$TARGET" + echo "Removed $TARGET" + else + echo "No directory to clean" + fi + + echo "Removing temporary log files..." + rm -rf "$LOG_DIR" + #rm -f "${LOG_DIR}.tar.gz" || true + echo "Cleanup completed" \ No newline at end of file diff --git a/.github/workflows/current-arm9-daily-build-devel.yml b/.github/workflows/current-arm9-daily-build-devel.yml new file mode 100644 index 00000000..f1744529 --- /dev/null +++ b/.github/workflows/current-arm9-daily-build-devel.yml @@ -0,0 +1,271 @@ +name: Current Daily Build Devel - arm9 + +# Define default environment variables for carrying defaults at the top for easy modification +env: + DEFAULT_CLI_BRANCH: "REL25_01" # Default CLI branch for scheduled runs + DEFAULT_MODE: "current" # Always "current" for this workflow + DEFAULT_COMPONENT: "spock50" # Default spock component name + DEFAULT_BRANCH: "main" # Default branch for the spock component + DEFAULT_CLEAN_FLAG: "false" # Default clean flag for scheduled runs + +# Triggers: Scheduled at 12:30 AM UTC and manual workflow dispatch +on: + workflow_dispatch: + inputs: + cli_branch: + description: "Select the CLI branch to build from (e.g. REL25_01)" + required: true + default: "REL25_01" + type: choice + options: + - REL25_01 + - REL24_10 + + component: + description: "Spock in-dev component to additionally build (e.g. spock50)" + required: true + default: "spock50" + type: string + + branch: + description: "Branch to use for spock component (e.g. main)" + required: true + default: "main" + type: string + + prefix_selector: + description: "Choose S3 subdirectory strategy (default: MMDD, or custom)" + required: true + default: "default" + type: choice + options: + - default + - custom + + custom_prefix: + description: "If 'custom' selected above, provide sub-directory name" + required: false + type: string + + clean_prefix: + description: "Clean the repo sub-directory before copying packages" + required: false + type: choice + options: + - "false" + - "true" + default: "false" + + schedule: + - cron: "30 0 * * *" # 12:30 AM UTC, 8:30pm EST.. 1 hour after stable build at 11:30 PM UTC + +jobs: + build-current-arm9: + runs-on: [self-hosted, arm9] + + steps: + # Resolve parameters (scheduled vs manual) and set up timestamped log directory + - name: Set build parameters and logfile + id: vars + run: | + source ~/.bashrc + + echo "Resolving build parameters for CURRENT mode workflow..." + + TODAY=$(date +%m%d) + RUNNER_NAME="${{ runner.name }}" + + # Create timestamped log directory with mmddhhmmss format + TIMESTAMP=$(date +%m%d%H%M%S) + LOG_DIR="/tmp/current-build-$TIMESTAMP" + mkdir -p "$LOG_DIR" + echo "Created log directory: $LOG_DIR" + + # Determine parameters based on trigger type + if [[ "${{ github.event_name }}" == "schedule" ]]; then + # Scheduled run: use defaults + PREFIX="$TODAY" + CLEAN_FLAG="" + SELECTED_CLI_BRANCH="${DEFAULT_CLI_BRANCH}" + COMPONENT="${DEFAULT_COMPONENT}" + BRANCH="${DEFAULT_BRANCH}" + echo "Scheduled run: Prefix=$PREFIX, Component=$COMPONENT, Branch=$BRANCH, Clean flag not set, CLI Branch=$SELECTED_CLI_BRANCH" + + else + # Manual run: process inputs + if [[ "${{ inputs.prefix_selector }}" == "default" ]]; then + PREFIX="$TODAY" + echo "Manual run: Using default prefix: $PREFIX" + else + # Custom prefix validation + if [[ "${{ inputs.prefix_selector }}" == "custom" && -z "${{ inputs.custom_prefix }}" ]]; then + echo "Error: Custom prefix selected but no input provided." + exit 1 + fi + PREFIX="${{ inputs.custom_prefix }}" + echo "Manual run: Using custom prefix: $PREFIX" + fi + + # Clean flag handling + if [[ "${{ inputs.clean_prefix }}" == "true" ]]; then + CLEAN_FLAG="-x" + echo "Clean flag is enabled" + else + CLEAN_FLAG="" + echo "Clean flag is disabled" + fi + + SELECTED_CLI_BRANCH="${{ inputs.cli_branch }}" + COMPONENT="${{ inputs.component }}" + BRANCH="${{ inputs.branch }}" + echo "Manual run: Component=$COMPONENT, Branch=$BRANCH, CLI Branch=$SELECTED_CLI_BRANCH" + fi + + # Set logfile path, matching stable workflow with "current" label + LOGFILE_NAME="build_to_devel_current-${RUNNER_NAME}-${TIMESTAMP}.log" + LOGFILE_PATH="$LOG_DIR/$LOGFILE_NAME" + + # Export variables + echo "RUNNER_NAME=$RUNNER_NAME" >> $GITHUB_ENV + echo "MODE=${DEFAULT_MODE}" >> $GITHUB_ENV + echo "COMPONENT=$COMPONENT" >> $GITHUB_ENV + echo "BRANCH=$BRANCH" >> $GITHUB_ENV + echo "PREFIX=$PREFIX" >> $GITHUB_ENV + echo "CLEAN_FLAG=$CLEAN_FLAG" >> $GITHUB_ENV + echo "LOG_DIR=$LOG_DIR" >> $GITHUB_ENV + echo "LOGFILE_PATH=$LOGFILE_PATH" >> $GITHUB_ENV + echo "CLI_BRANCH=$SELECTED_CLI_BRANCH" >> $GITHUB_ENV + + echo "Log file will be created at: $LOGFILE_PATH" + + # Display the resolved configuration for debugging + - name: Show effective configuration + run: | + echo "=== CURRENT BUILD WORKFLOW CONFIGURATION ===" + echo "Runner: $RUNNER_NAME" + echo "Mode: $MODE" + echo "Component: $COMPONENT" + echo "Branch: $BRANCH" + echo "S3 Prefix: $PREFIX" + echo "Clean flag: $CLEAN_FLAG" + echo "Log directory: $LOG_DIR" + echo "Log file path: $LOGFILE_PATH" + echo "CLI branch: $CLI_BRANCH" + echo "==============================================" + + # Verify environment details and switch to the specified CLI branch + - name: Confirm environment and switch CLI branch + id: cli_branch_checkout + run: | + source ~/.bashrc + echo "OS info:" + cat /etc/os-release + echo "Architecture:" + arch + + echo "Navigating to CLI directory: $PGE" + cd $PGE + + echo "Current branch and status:" + git branch + git status + + echo "Stashing changes if any..." + git stash push -u -m "Temp stash for current build" || true + + echo "Switching to branch: $CLI_BRANCH" + git checkout "$CLI_BRANCH" || true + echo "Branch checkout completed" + + # Execute the build script and collect artifacts into src/ and bin/ + - name: Run build and push to devel repo + id: build_to_devel + run: | + source ~/.bashrc + + echo "Launching build script in CURRENT mode..." + cd $DEV + + # Construct and run the build command + BUILD_CMD="./build_to_devel_2.sh -m $MODE -c $COMPONENT -b $BRANCH -d $PREFIX" + if [[ -n "$CLEAN_FLAG" ]]; then + BUILD_CMD="$BUILD_CMD $CLEAN_FLAG" + fi + + echo "Executing: $BUILD_CMD" + $BUILD_CMD > "$LOGFILE_PATH" 2>&1 + EXIT_CODE=$? + + echo "Build finished with exit code: $EXIT_CODE" + + # Collect artifacts into src/ and bin/ + echo "Collecting source and binary artifacts..." + mkdir -p "$LOG_DIR/src" "$LOG_DIR/bin" + + echo "Copying source files from $SOURCE/${COMPONENT}*" + if ls "$SOURCE/${COMPONENT}"* 1> /dev/null 2>&1; then + cp "$SOURCE/${COMPONENT}"* "$LOG_DIR/src/" 2>/dev/null || true + echo "Source files copied" + else + echo "No source files found" + fi + + echo "Copying binaries from $IN/postgres/$COMPONENT/${COMPONENT}*" + if ls "$IN/postgres/$COMPONENT/${COMPONENT}"* 1> /dev/null 2>&1; then + cp "$IN/postgres/$COMPONENT/${COMPONENT}"* "$LOG_DIR/bin/" 2>/dev/null || true + echo "Binary files copied" + else + echo "No binary files found" + fi + + # List collected artifacts + echo "=== COLLECTED ARTIFACTS ===" + ls -la "$LOG_DIR/" + ls -la "$LOG_DIR/src/" || echo "No source files" + ls -la "$LOG_DIR/bin/" || echo "No binary files" + echo "==========================" + + if [[ $EXIT_CODE -ne 0 ]]; then + echo "Build failed. Artifacts collected for debugging." + exit $EXIT_CODE + fi + + # Compress the log directory for artifact upload + - name: Compress log directory + if: always() + run: | + echo "Compressing log directory: $LOG_DIR" + cd "$(dirname "$LOG_DIR")" + tar -czf "${LOG_DIR}.tar.gz" "$(basename "$LOG_DIR")" + echo "Compressed archive created: ${LOG_DIR}.tar.gz" + + # Upload the compressed log archive as an artifact + - name: Upload build log artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: current-build-log + path: ${{ env.LOG_DIR }}.tar.gz + if-no-files-found: warn + + # Clean up historical build artifacts and temporary files + - name: Cleanup build artifacts + if: steps.build_to_devel.outcome == 'success' + continue-on-error: true + run: | + source ~/.bashrc + + run_day=$(date +%j) + TARGET="$HIST/devel-$run_day" + echo "Cleaning up historical build directory: $TARGET" + if [ -d "$TARGET" ]; then + rm -rf "$TARGET" + echo "Removed $TARGET" + else + echo "No directory to clean" + fi + + echo "Removing temporary log files..." + rm -rf "$LOG_DIR" + #rm -f "${LOG_DIR}.tar.gz" || true + echo "Cleanup completed" \ No newline at end of file From 7eaee4f8a46a0b5fa8ad91f4bffc8e5902fe9706 Mon Sep 17 00:00:00 2001 From: Hayee Bhatti <152845623+hayee-bhatti@users.noreply.github.com> Date: Fri, 13 Jun 2025 15:52:47 +0500 Subject: [PATCH 06/42] Updating the workflows to reflect the new cli branches v25_STABLE and main (#330) --- .github/workflows/current-amd8-daily-build-devel.yml | 7 ++++--- .github/workflows/current-arm9-daily-build-devel.yml | 9 +++++---- .github/workflows/stable-amd8-daily-build-devel.yml | 7 ++++--- .github/workflows/stable-arm9-daily-build-devel.yml | 7 ++++--- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.github/workflows/current-amd8-daily-build-devel.yml b/.github/workflows/current-amd8-daily-build-devel.yml index c1834e1d..a104372d 100644 --- a/.github/workflows/current-amd8-daily-build-devel.yml +++ b/.github/workflows/current-amd8-daily-build-devel.yml @@ -2,7 +2,7 @@ name: Current Daily Build Devel - amd8 # Define default environment variables for carrying defaults at the top for easy modification env: - DEFAULT_CLI_BRANCH: "REL25_01" # Default CLI branch for scheduled runs + DEFAULT_CLI_BRANCH: "v25_STABLE" # Default CLI branch for scheduled runs DEFAULT_MODE: "current" # Always "current" for this workflow DEFAULT_COMPONENT: "spock50" # Default spock component name DEFAULT_BRANCH: "main" # Default branch for the spock component @@ -15,10 +15,11 @@ on: cli_branch: description: "Select the CLI branch to build from (e.g. REL25_01)" required: true - default: "REL25_01" + default: "v25_STABLE" type: choice options: - - REL25_01 + - v25_STABLE + - main - REL24_10 component: diff --git a/.github/workflows/current-arm9-daily-build-devel.yml b/.github/workflows/current-arm9-daily-build-devel.yml index f1744529..7cecaddd 100644 --- a/.github/workflows/current-arm9-daily-build-devel.yml +++ b/.github/workflows/current-arm9-daily-build-devel.yml @@ -2,7 +2,7 @@ name: Current Daily Build Devel - arm9 # Define default environment variables for carrying defaults at the top for easy modification env: - DEFAULT_CLI_BRANCH: "REL25_01" # Default CLI branch for scheduled runs + DEFAULT_CLI_BRANCH: "v25_STABLE" # Default CLI branch for scheduled runs DEFAULT_MODE: "current" # Always "current" for this workflow DEFAULT_COMPONENT: "spock50" # Default spock component name DEFAULT_BRANCH: "main" # Default branch for the spock component @@ -13,12 +13,13 @@ on: workflow_dispatch: inputs: cli_branch: - description: "Select the CLI branch to build from (e.g. REL25_01)" + description: "Select the CLI branch to build from (e.g. v25_STABLE)" required: true - default: "REL25_01" + default: "v25_STABLE" type: choice options: - - REL25_01 + - v25_STABLE + - main - REL24_10 component: diff --git a/.github/workflows/stable-amd8-daily-build-devel.yml b/.github/workflows/stable-amd8-daily-build-devel.yml index aaa27e56..c8bb9887 100644 --- a/.github/workflows/stable-amd8-daily-build-devel.yml +++ b/.github/workflows/stable-amd8-daily-build-devel.yml @@ -1,7 +1,7 @@ name: Stable Daily Build Devel - amd8 env: - DEFAULT_CLI_BRANCH: "REL25_01" + DEFAULT_CLI_BRANCH: "v25_STABLE" on: workflow_dispatch: @@ -9,10 +9,11 @@ on: cli_branch: description: "Select the CLI branch to build from" required: true - default: REL25_01 + default: v25_STABLE type: choice options: - - REL25_01 + - v25_STABLE + - main - REL24_10 prefix_selector: diff --git a/.github/workflows/stable-arm9-daily-build-devel.yml b/.github/workflows/stable-arm9-daily-build-devel.yml index 1d224fbb..c26fdfcb 100644 --- a/.github/workflows/stable-arm9-daily-build-devel.yml +++ b/.github/workflows/stable-arm9-daily-build-devel.yml @@ -1,7 +1,7 @@ name: Stable Daily Build Devel - arm9 env: - DEFAULT_CLI_BRANCH: "REL25_01" + DEFAULT_CLI_BRANCH: "v25_STABLE" on: workflow_dispatch: @@ -9,10 +9,11 @@ on: cli_branch: description: "Select the CLI branch to build from" required: true - default: REL25_01 + default: v25_STABLE type: choice options: - - REL25_01 + - v25_STABLE + - main - REL24_10 prefix_selector: From 5786f55d0da5707cb206e1ef12d22a045f97fd79 Mon Sep 17 00:00:00 2001 From: Hayee Bhatti <152845623+hayee-bhatti@users.noreply.github.com> Date: Fri, 13 Jun 2025 17:22:30 +0500 Subject: [PATCH 07/42] Fixing an issue with the script name (#331) --- .github/workflows/current-amd8-daily-build-devel.yml | 4 ++-- .github/workflows/current-arm9-daily-build-devel.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/current-amd8-daily-build-devel.yml b/.github/workflows/current-amd8-daily-build-devel.yml index a104372d..40ddcebd 100644 --- a/.github/workflows/current-amd8-daily-build-devel.yml +++ b/.github/workflows/current-amd8-daily-build-devel.yml @@ -13,7 +13,7 @@ on: workflow_dispatch: inputs: cli_branch: - description: "Select the CLI branch to build from (e.g. REL25_01)" + description: "Select the CLI branch to build from (e.g. v25_STABLE)" required: true default: "v25_STABLE" type: choice @@ -188,7 +188,7 @@ jobs: cd $DEV # Construct and run the build command - BUILD_CMD="./build_to_devel_2.sh -m $MODE -c $COMPONENT -b $BRANCH -d $PREFIX" + BUILD_CMD="./build_to_devel.sh -m $MODE -c $COMPONENT -b $BRANCH -d $PREFIX" if [[ -n "$CLEAN_FLAG" ]]; then BUILD_CMD="$BUILD_CMD $CLEAN_FLAG" fi diff --git a/.github/workflows/current-arm9-daily-build-devel.yml b/.github/workflows/current-arm9-daily-build-devel.yml index 7cecaddd..9b691cc1 100644 --- a/.github/workflows/current-arm9-daily-build-devel.yml +++ b/.github/workflows/current-arm9-daily-build-devel.yml @@ -188,7 +188,7 @@ jobs: cd $DEV # Construct and run the build command - BUILD_CMD="./build_to_devel_2.sh -m $MODE -c $COMPONENT -b $BRANCH -d $PREFIX" + BUILD_CMD="./build_to_devel.sh -m $MODE -c $COMPONENT -b $BRANCH -d $PREFIX" if [[ -n "$CLEAN_FLAG" ]]; then BUILD_CMD="$BUILD_CMD $CLEAN_FLAG" fi From cd376e5b94603a9169391433d1384dd6188aecda Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Wed, 18 Jun 2025 08:27:09 -0500 Subject: [PATCH 08/42] remove static pages gh action (#334) --- .github/workflows/static.yml | 42 ------------------------------------ 1 file changed, 42 deletions(-) delete mode 100644 .github/workflows/static.yml diff --git a/.github/workflows/static.yml b/.github/workflows/static.yml deleted file mode 100644 index c790867e..00000000 --- a/.github/workflows/static.yml +++ /dev/null @@ -1,42 +0,0 @@ -# Simple workflow for deploying static content to GitHub Pages -name: Deploy static content to Pages - -on: - # Runs on pushes targeting the default branch - push: - branches: ["main"] - - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - pages: write - id-token: write - -# Allow one concurrent deployment -concurrency: - group: "pages" - cancel-in-progress: true - -jobs: - # Single deploy job since we're just deploying - deploy: - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Setup Pages - uses: actions/configure-pages@v3 - - name: Upload artifact - uses: actions/upload-pages-artifact@v1 - with: - # Upload entire repository - path: '.' - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v1 From 62587994c6d6e23da584607ce0e4ca71c98b580b Mon Sep 17 00:00:00 2001 From: Hayee Bhatti <152845623+hayee-bhatti@users.noreply.github.com> Date: Thu, 19 Jun 2025 02:23:04 +0500 Subject: [PATCH 09/42] Workflow branch and execution time adjustments (#336) - Changing default cli branch to main for current workflow - Adjusting time slightly to not produce builds in the next day MMDD in s3 --- .github/workflows/current-amd8-daily-build-devel.yml | 8 ++++---- .github/workflows/current-arm9-daily-build-devel.yml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/current-amd8-daily-build-devel.yml b/.github/workflows/current-amd8-daily-build-devel.yml index 40ddcebd..76a334ee 100644 --- a/.github/workflows/current-amd8-daily-build-devel.yml +++ b/.github/workflows/current-amd8-daily-build-devel.yml @@ -2,7 +2,7 @@ name: Current Daily Build Devel - amd8 # Define default environment variables for carrying defaults at the top for easy modification env: - DEFAULT_CLI_BRANCH: "v25_STABLE" # Default CLI branch for scheduled runs + DEFAULT_CLI_BRANCH: "main" # Default CLI branch for scheduled runs DEFAULT_MODE: "current" # Always "current" for this workflow DEFAULT_COMPONENT: "spock50" # Default spock component name DEFAULT_BRANCH: "main" # Default branch for the spock component @@ -15,11 +15,11 @@ on: cli_branch: description: "Select the CLI branch to build from (e.g. v25_STABLE)" required: true - default: "v25_STABLE" + default: "main" type: choice options: - - v25_STABLE - main + - v25_STABLE - REL24_10 component: @@ -58,7 +58,7 @@ on: default: "false" schedule: - - cron: "0 0 * * *" # 12:00 AM UTC, 8pm EST.. 1 hour after stable build at 11:00 PM UTC + - cron: "40 23 * * *" # 11:40 PM UTC. jobs: build-current-amd8: diff --git a/.github/workflows/current-arm9-daily-build-devel.yml b/.github/workflows/current-arm9-daily-build-devel.yml index 9b691cc1..f77a092d 100644 --- a/.github/workflows/current-arm9-daily-build-devel.yml +++ b/.github/workflows/current-arm9-daily-build-devel.yml @@ -2,7 +2,7 @@ name: Current Daily Build Devel - arm9 # Define default environment variables for carrying defaults at the top for easy modification env: - DEFAULT_CLI_BRANCH: "v25_STABLE" # Default CLI branch for scheduled runs + DEFAULT_CLI_BRANCH: "main" # Default CLI branch for scheduled runs DEFAULT_MODE: "current" # Always "current" for this workflow DEFAULT_COMPONENT: "spock50" # Default spock component name DEFAULT_BRANCH: "main" # Default branch for the spock component @@ -15,11 +15,11 @@ on: cli_branch: description: "Select the CLI branch to build from (e.g. v25_STABLE)" required: true - default: "v25_STABLE" + default: "main" type: choice options: - - v25_STABLE - main + - v25_STABLE - REL24_10 component: @@ -58,7 +58,7 @@ on: default: "false" schedule: - - cron: "30 0 * * *" # 12:30 AM UTC, 8:30pm EST.. 1 hour after stable build at 11:30 PM UTC + - cron: "50 23 * * *" # 11:50 PM UTC jobs: build-current-arm9: From 0b28a7fbdbedc353af56627b9e3d2835b441c277 Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Thu, 19 Jun 2025 17:37:26 +0500 Subject: [PATCH 10/42] [BR-81]: Update 10-toolset.sh to install new build dependency for spock (#337) --- devel/setup/10-toolset.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devel/setup/10-toolset.sh b/devel/setup/10-toolset.sh index 24e1d6ca..c7ef4089 100755 --- a/devel/setup/10-toolset.sh +++ b/devel/setup/10-toolset.sh @@ -65,7 +65,7 @@ if [ $el_supported == "yes" ]; then sudo dnf -y --nobest install zlib-devel bzip2-devel lbzip2 \ openssl-devel libxslt-devel libevent-devel c-ares-devel \ perl-ExtUtils-Embed pam-devel openldap-devel boost-devel - $yum curl-devel + $yum curl-devel jansson-devel $yum chrpath clang-devel llvm-devel cmake libxml2-devel $yum libedit-devel $yum *ossp-uuid* From cd41065d36c39473c0e7c3da39e35990cc1a0cf0 Mon Sep 17 00:00:00 2001 From: hayee-bhatti Date: Thu, 19 Jun 2025 13:30:37 +0000 Subject: [PATCH 11/42] Bumped pgV build numbers to -2 to include a new dependency --- env.sh | 6 +++--- src/conf/versions.sql | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/env.sh b/env.sh index bd119765..0acb87eb 100755 --- a/env.sh +++ b/env.sh @@ -21,13 +21,13 @@ removeComponentFromOut=spock50 lolorV=1.2-1 snwflkV=2.2-1 -P17=17.5-1 +P17=17.5-2 P171=17.0-1 -P16=16.9-1 +P16=16.9-2 P161=16.4-2 -P15=15.13-1 +P15=15.13-2 P151=15.8-2 vectorV=0.8.0-1 diff --git a/src/conf/versions.sql b/src/conf/versions.sql index ca104837..953c5393 100644 --- a/src/conf/versions.sql +++ b/src/conf/versions.sql @@ -156,17 +156,17 @@ INSERT INTO projects VALUES ('pg', 'pge', 1, 5432, '', 1, 'https://github.com/po INSERT INTO releases VALUES ('pg15', 2, 'pg', '', '', 'prod', 'New in 2022', 1, 'POSTGRES', '', ''); -INSERT INTO versions VALUES ('pg15', '15.13-1', 'amd, arm', 1, '20250508','', '', ''); +INSERT INTO versions VALUES ('pg15', '15.13-2', 'amd, arm', 1, '20250619','', '', ''); INSERT INTO versions VALUES ('pg15', '15.12-1', 'amd, arm', 0, '20250224','', '', ''); INSERT INTO releases VALUES ('pg16', 2, 'pg', '', '', 'prod', 'New in 2023!', 1, 'POSTGRES', '', ''); -INSERT INTO versions VALUES ('pg16', '16.9-1', 'amd, arm', 1, '20250508','', '', ''); +INSERT INTO versions VALUES ('pg16', '16.9-2', 'amd, arm', 1, '20250619','', '', ''); INSERT INTO versions VALUES ('pg16', '16.8-1', 'amd, arm', 0, '20250224','', '', ''); INSERT INTO releases VALUES ('pg17', 2, 'pg', '', '', 'prod', 'New in 2024!', 1, 'POSTGRES', '', ''); -INSERT INTO versions VALUES ('pg17', '17.5-1', 'amd, arm', 1, '20250508','', '', ''); +INSERT INTO versions VALUES ('pg17', '17.5-2', 'amd, arm', 1, '20250619','', '', ''); INSERT INTO versions VALUES ('pg17', '17.4-1', 'amd, arm', 0, '20250224','', '', ''); -- ## ORAFCE ############################# From 26a8f7c7773e38da8afd0d0e9e9dee88232ca606 Mon Sep 17 00:00:00 2001 From: hayee-bhatti Date: Mon, 23 Jun 2025 18:29:43 +0000 Subject: [PATCH 12/42] Fixed a PG server version for spock50-pg15 in versions.sql --- src/conf/versions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/conf/versions.sql b/src/conf/versions.sql index 953c5393..1b220420 100644 --- a/src/conf/versions.sql +++ b/src/conf/versions.sql @@ -353,7 +353,7 @@ INSERT INTO releases VALUES ('spock50-pg15', 4, 'spock', 'Spock', '', 'test', '' INSERT INTO releases VALUES ('spock50-pg16', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); INSERT INTO releases VALUES ('spock50-pg17', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); -INSERT INTO versions VALUES ('spock50-pg15', '5.0.0-devel1-1', 'amd, arm', 1, '20250521', 'pg16', '', ''); +INSERT INTO versions VALUES ('spock50-pg15', '5.0.0-devel1-1', 'amd, arm', 1, '20250521', 'pg15', '', ''); INSERT INTO versions VALUES ('spock50-pg16', '5.0.0-devel1-1', 'amd, arm', 1, '20250521', 'pg16', '', ''); INSERT INTO versions VALUES ('spock50-pg17', '5.0.0-devel1-1', 'amd, arm', 1, '20250521', 'pg17', '', ''); From cd591233226b0d828a65de4069cbf00f0dc51037 Mon Sep 17 00:00:00 2001 From: asifnaeem Date: Thu, 26 Jun 2025 01:07:39 +0500 Subject: [PATCH 13/42] Support custom data directory via pg_data flag (#324) * CLI feature -datadir to pass absolute path for data directory to pgedge setup and cluster commands * adjustments based on review (PR#324) * more adjustments based on review (PR#324) * more adjustments based on review (PR#324) * done some cleanup (PR#324) * ensure fallback for pg_data * fix helptext in setup --------- Co-authored-by: Matthew Mols --- cli/scripts/cluster.py | 50 +++++++++++++++++++++++++++++---------- cli/scripts/setup.py | 20 ++++++++++++---- cli/scripts/setup_core.py | 19 ++++++++------- docs/functions/setup.md | 5 +++- src/pgXX/init-pgXX.py | 2 +- 5 files changed, 69 insertions(+), 27 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index d6a1fa42..a4915c4a 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -195,6 +195,10 @@ def load_json(cluster_name): os_user = ssh_info.get("os_user", "") ssh_key = ssh_info.get("private_key", "") + pg_data = group.get("pg_data", "") + if not pg_data: + pg_data = group.get("path") + "/pgedge/data/pg" + db_settings["pg_version"] + node_info = { "name": group.get("name", ""), "is_active": group.get("is_active", ""), @@ -202,6 +206,7 @@ def load_json(cluster_name): "private_ip": group.get("private_ip", ""), "port": group.get("port", ""), "path": group.get("path", ""), + "pg_data": pg_data, "os_user": os_user, "ssh_key": ssh_key, "backrest": group.get("backrest", {}), @@ -214,6 +219,11 @@ def load_json(cluster_name): # Process sub_nodes sub_node_list = [] for sub_node in group.get("sub_nodes", []): + + sub_pg_data = sub_node.get("pg_data", "") + if not sub_pg_data: + sub_pg_data = sub_node.get("path") + "/pgedge/data/pg" + db_settings["pg_version"] + sub_node_info = { "name": sub_node.get("name", ""), "is_active": sub_node.get("is_active", ""), @@ -221,6 +231,7 @@ def load_json(cluster_name): "private_ip": sub_node.get("private_ip", ""), "port": sub_node.get("port", ""), "path": sub_node.get("path", ""), + "pg_data": sub_pg_data, "os_user": os_user, "ssh_key": ssh_key, } @@ -363,13 +374,20 @@ def save_updated_json(cluster_name, updated_json): # Validate and update node_groups for idx, node in enumerate(parsed_json.get("node_groups", [])): summary["total_nodes"] += 1 # Increment total node count + path = node.get("path", "/var/lib/postgresql") + pg_data = node.get("pg_data", "") + if pg_data and not os.path.isabs(pg_data): + util.exit_message( + "pg_data cannot be set as relative path. Please specify absolute path or leave it blank." + ) node_info = { "node_index": idx + 1, # Start numbering nodes from 1 "public_ip": node.get("public_ip", ""), "private_ip": node.get("private_ip", ""), "port": node.get("port", 5432), "is_active": node.get("is_active", "off"), - "path": node.get("path", "/var/lib/postgresql"), + "path": path, + "pg_data": pg_data, } # Validate subnodes @@ -513,7 +531,6 @@ def ssh_install_pgedge( install (bool): Whether or not to perform 'pgedge install'. verbose (bool): Whether to produce verbose output. """ - if install is None: install = True @@ -527,6 +544,7 @@ def ssh_install_pgedge( ndnm = n["name"] ndpath = n["path"] + nddatadir = n["pg_data"] ndip = n["public_ip"] or n["private_ip"] ndport = str(n.get("port", "5432")) pg = db_settings["pg_version"] @@ -563,6 +581,8 @@ def ssh_install_pgedge( setup_parms += f" --spock_ver {spock}" if db_settings.get("auto_start") == "on": setup_parms += " --autostart" + if nddatadir: + setup_parms += f" --pg_data {nddatadir}" cmd_setup = f"{nc} setup {setup_parms}" message = f"Setting up pgEdge on {ndnm}" @@ -999,6 +1019,7 @@ def get_cluster_info(cluster_name): node_json["port"] = str(node_port) node_json["path"] = f"/home/{os_user}/{cluster_name}/n{n}" + node_json["pg_data"] = f"{node_json['path']}/pgedge/data/pg{pg_version_int}" # Update backrest configuration to always append the node name to the repo1_path. if backrest_enabled: @@ -1379,8 +1400,7 @@ def init(cluster_name, install=True): if not restore_path.rstrip("/").endswith(node["name"]): restore_path = restore_path.rstrip("/") + f"/{node['name']}" - pg_version = db_settings["pg_version"] - pg1_path = f"{node['path']}/pgedge/data/pg{pg_version}" + pg1_path = node.get("pg_data") port = node["port"] # Custom port from JSON # Install pgBackRest @@ -1666,6 +1686,11 @@ def add_node( os_user = ssh_info.get("os_user", "") ssh_key = ssh_info.get("private_key", "") + # Fallback to /pgedge/data/pgV if pg_data is missing or empty + pg_data = group.get("pg_data", "") + if not pg_data: + pg_data = f"{group.get('path', '')}/pgedge/data/{pgV}" + target_node_data = { "ssh": ssh_info, "backrest": group.get("backrest", {}), @@ -1675,6 +1700,7 @@ def add_node( "private_ip": group.get("private_ip", ""), "port": group.get("port", ""), "path": group.get("path", ""), + "pg_data": pg_data, "os_user": os_user, "ssh_key": ssh_key, } @@ -1725,7 +1751,7 @@ def add_node( ) pg_version = db_settings["pg_version"] - source_pg1_path = f"{source_node_data['path']}/pgedge/data/pg{pg_version}" + source_pg1_path = source_node_data["pg_data"] source_port = source_node_data["port"] # Configure postgresql.conf for pgBackRest (without --pg1-port) @@ -1872,7 +1898,7 @@ def add_node( repo1_path_default = f"/var/lib/pgbackrest/{source_node_data['name']}" repo1_path = source_backrest_cfg.get("repo1_path", f"{repo1_path_default}") else: - pg1_path = f"{source_node_data['path']}/pgedge/data/{pgV}" + pg1_path = source_node_data['pg_data'] cmd = ( f"{source_node_data['path']}/pgedge/pgedge backrest set-postgresqlconf {source_stanza} " @@ -1895,7 +1921,7 @@ def add_node( run_cmd(cmd, target_node_data, message=message, verbose=verbose) manage_node(target_node_data, "stop", f"{pgV}", verbose) - cmd = f'rm -rf {target_node_data["path"]}/pgedge/data/{pgV}' + cmd = f'rm -rf {target_node_data["pg_data"]}' message = f"Removing old data directory" run_cmd(cmd, target_node_data, message=message, verbose=verbose) @@ -1904,7 +1930,7 @@ def add_node( # by default when generating the restore_command f'--cmd="pgbackrest --repo1-cipher-type={source_repo1_cipher_type}" ' f"--stanza={source_stanza} " - f"--pg1-path={target_node_data['path']}/pgedge/data/{pgV} " + f"--pg1-path={target_node_data['pg_data']} " f"--repo1-path={repo1_path} " f"--repo1-cipher-type={source_repo1_cipher_type} " f"--repo1-type={source_repo1_type} " @@ -1922,7 +1948,7 @@ def add_node( message = f"Restoring backup" run_cmd(cmd, target_node_data, message=message, verbose=verbose) - pgd = f'{target_node_data["path"]}/pgedge/data/{pgV}' + pgd = target_node_data["pg_data"] pgc = f"{pgd}/postgresql.conf" log_directory = f'{target_node_data["path"]}/pgedge/data/logs/{pgV}' @@ -1941,7 +1967,7 @@ def add_node( # Step 5. Configure the target node as a standby replica of the source node. cmd = ( f'{target_node_data["path"]}/pgedge/pgedge backrest configure-replica {source_stanza} ' - f'{target_node_data["path"]}/pgedge/data/{pgV} {source_node_data.get("private_ip", source_node_data.get("public_ip"))} ' + f'{target_node_data["pg_data"]} {source_node_data.get("private_ip", source_node_data.get("public_ip"))} ' f'{source_node_data["port"]} {source_node_data["os_user"]}' ) message = f"Configuring PITR on replica" @@ -2117,7 +2143,7 @@ def add_node( cmd = ( f"cd {target_node_data['path']}/pgedge && " f"./pgedge backrest cleanup-replica " - f"--pg1-path {target_node_data['path']}/pgedge/data/{pgV} " + f"--pg1-path {target_node_data['pg_data']} " ) run_cmd( cmd, @@ -2146,7 +2172,7 @@ def add_node( "repo1_host_user", target_node_data.get("os_user", "postgres") ) target_pg1_path = target_backrest_cfg.get( - "pg1_path", f"{target_pgedge_dir}/data/{pgV}" + "pg1_path", target_node_data['pg_data'] ) target_pg1_user = target_backrest_cfg.get( "pg1_user", target_node_data.get("os_user", "postgres") diff --git a/cli/scripts/setup.py b/cli/scripts/setup.py index 891462ce..2fc21aa9 100755 --- a/cli/scripts/setup.py +++ b/cli/scripts/setup.py @@ -11,16 +11,17 @@ CTL="./pgedge" -def setup_pgedge(User=None, Passwd=None, dbName=None, port=None, pg_ver=None, spock_ver=None, autostart=False, interactive=False, yes=False): +def setup_pgedge(User=None, Passwd=None, dbName=None, port=None, pg_data=None, pg_ver=None, spock_ver=None, autostart=False, interactive=False, yes=False): """Install a pgEdge node (including PostgreSQL, spock, and snowflake-sequences). Install a pgEdge node (including PostgreSQL, spock, and snowflake-sequences). - Example: ./pgedge setup -U user -P passwd -d test --pg_ver 16 + Example: ./pgedge setup -U admin -P passwd -d defaultdb --pg_ver 16 :param User: The database user that will own the db (required) :param Passwd: The password for the newly created db user (required) :param dbName: The database name (required) :param port: Defaults to 5432 if not specified + :param pg_data: The data directory to use for PostgreSQL. Must be an absolute path. Defaults to data/pgV, relative to where the CLI is installed :param pg_ver: Defaults to latest prod version of pg, such as 16. May be pinned to a specific pg version such as 16.4 :param spock_ver: Defaults to latest prod version of spock, such as 4.0. May be pinned to a specific spock version such as 4.0.1 :param autostart: Defaults to False @@ -54,7 +55,7 @@ def setup_pgedge(User=None, Passwd=None, dbName=None, port=None, pg_ver=None, sp port = pgePort util.message(f""" -setup.pgedge(User={User}, Passwd={Passwd}, dbName={dbName}, port={port}, pg_ver={pg_ver}, +setup.pgedge(User={User}, Passwd={Passwd}, dbName={dbName}, port={port}, pg_data={pg_data}, pg_ver={pg_ver}, spock_ver={spock_ver}, autostart={autostart}, interactive={interactive}, yes={yes}) """, "debug") @@ -95,8 +96,17 @@ def setup_pgedge(User=None, Passwd=None, dbName=None, port=None, pg_ver=None, sp pg_major, pg_minor = setup_core.parse_pg(pg_ver) + pg_init_options = "" + if pg_data is not None: + pg_data = pg_data.rstrip("/") + if not os.path.isabs(pg_data): + util.exit_message( + "pg_data cannot be set as relative path. Please specify absolute path instead" + ) + pg_init_options = f"--datadir={pg_data}" + setup_core.check_pre_reqs( - User, Passwd, dbName, port, pg_major, pg_minor, spock_ver, autostart) + User, Passwd, dbName, port, pg_data, pg_major, pg_minor, spock_ver, autostart) if interactive and yes is False: y_or_n = input("Do you want to continue? [Y/n] ") @@ -119,7 +129,7 @@ def setup_pgedge(User=None, Passwd=None, dbName=None, port=None, pg_ver=None, sp if autostart is True: util.autostart_config(pg_maj) else: - setup_core.osSys(f"{CTL} init {pg_maj}") + setup_core.osSys(f"{CTL} init {pg_maj} {pg_init_options}") setup_core.osSys(f"{CTL} config {pg_maj} --port={port}") diff --git a/cli/scripts/setup_core.py b/cli/scripts/setup_core.py index 97228e38..f5c91618 100755 --- a/cli/scripts/setup_core.py +++ b/cli/scripts/setup_core.py @@ -19,7 +19,7 @@ def osSys(cmd, fatal_exit=True, is_silent=False): def check_pre_reqs( - User, Passwd, db, port, pg_major, pg_minor, spock, autostart): + User, Passwd, db, port, data_dir, pg_major, pg_minor, spock, autostart): util.message( f"""setup_core.check_pre_reqs(User={User}, Passwd={Passwd}, db={db}, port={port}, @@ -62,7 +62,9 @@ def check_pre_reqs( util.exit_message( f"{num_pg_mins} versions available matching '{pg_minor}*'") - data_dir = f"data/pg{pg_major}" + if data_dir is None: + data_dir = f"{os.getcwd()}/data/pg{pg_major}" + if os.path.exists(data_dir): dir = os.listdir(data_dir) if len(dir) != 0: @@ -104,12 +106,13 @@ def check_pre_reqs( setup_info = f""" ######### pgEdge Setup Info ########### -# User: {User} -# Database: {db}:{port} -# Postgres: {pg_display} -# Spock: {spock_display} -# Autostart: {autostart} -# Platform: {util.get_ctlib_dir()} +# User: {User} +# Database: {db}:{port} +# Postgres: {pg_display} +# Data directory: {data_dir} +# Spock: {spock_display} +# Autostart: {autostart} +# Platform: {util.get_ctlib_dir()} ####################################### """ util.message(setup_info, "info") diff --git a/docs/functions/setup.md b/docs/functions/setup.md index ac0b1a26..a228ed48 100644 --- a/docs/functions/setup.md +++ b/docs/functions/setup.md @@ -5,7 +5,7 @@ ## DESCRIPTION Install a pgEdge node (including PostgreSQL, spock, and snowflake-sequences). -Example: ./pgedge setup -U user -P passwd -d test --pg_ver 16 +Example: ./pgedge setup -U admin -P passwd -d defaultdb --pg_ver 16 ## FLAGS -U, --User=USER @@ -20,6 +20,9 @@ Example: ./pgedge setup -U user -P passwd -d test --pg_ver 16 --port=PORT Defaults to 5432 if not specified + --pg_data=PG_DATA + The data directory to use for PostgreSQL. Must be an absolute path. Defaults to data/pgV, relative to where the CLI is installed + --pg_ver=PG_VER Defaults to latest prod version of pg, such as 16. May be pinned to a specific pg version such as 16.4 diff --git a/src/pgXX/init-pgXX.py b/src/pgXX/init-pgXX.py index 0b2b30ad..6111c7db 100644 --- a/src/pgXX/init-pgXX.py +++ b/src/pgXX/init-pgXX.py @@ -65,7 +65,7 @@ def fatal_error(p_msg): pg_data = args.datadir if not os.path.isdir(pg_data): - os.mkdir(pg_data) + os.makedirs(pg_data) ## SVCUSER ########################################### svcuser = "" From 040fce3eb7579efcabd7a94405b1fd8ec6823e92 Mon Sep 17 00:00:00 2001 From: Tej Kashi Date: Wed, 25 Jun 2025 16:17:40 -0400 Subject: [PATCH 14/42] Use Merkle trees in ACE (#253) * Added build-mtree to build a merkle tree for a table * Fix node_list population * Also minimise critical section in compare_checksums * Use same block ranges on all nodes * Breakthrough: extending a merkle tree on new inserts works * Sweeping optimisations in table-diff * Use an optimised sql query instead of get_pkey_offsets * Fix pkey offsets query * Added feature to merge blocks on large deletes * Handle blocks splits and merges inside ACE * Fix some rebalancing bugs * Integrate merkle trees into ACE cli * Update mtree only splits blocks * Merges happen only when --rebalance=true is passed * Adding mtree diff * First version of mtree diff ready * Add progress bar for splits and merges * Use always trigger for tracking dirty blocks * Update cryptography to address CVE * No longer using OrderedSet for comparisons * Handle mismatching tree levels * Use generic trigger functions * Initialise mtree objects once per DB node * More cleanup * Tweaks to get all base tests to pass * Use SQL composables; fix rebalance issues * Unify pkey offset computations * Add support for specifying a ranges file * Remove node len check for mtree build * Update leaf hash during mtree update * Temp fix for range boundary issue * Address boundary issues using a lookup table * Use prepared statements * Sunset batches option and async rerun * Add block boundary and repset-diff tests * Remove explicit stmt.close() * Add support for composite keys; use stmt triggers * SQL cleanup * Add mtree init, teardown; fix block size usage * Added merkle tree tests * Use mogrify during repairs * Prior use of executemany would internally call execute multiple times, thereby making repairs slow because of statement triggers from mtrees * Fix write-ranges to use str by default * Add support for non-numeric datatypes in tracking triggers * Fix conn establishment in cleanup * Address codacy issues * Addressed more codacy issues * String literal fix * Codacy fix #4 * Codacy fix #5 * Use nosemgrep * More nosemgreps * Add mtree cli helptext * Fix metadata task type * Add metadata tracking for mtree modules * Move error codes out of config file * Use a separate consts file * Revamp ACE CLI invocation * Group mtree cmds into a sub-cmd * Treat each of table-diff, -repair, etc. as a top-level sub-cmd * Update tests * Fix table names in mtree test * Minor fixes * Help texts mostly fixed * fix fire.py helptext generation for mtree submodule * generate help for ace mtree submodule * Backward compatibility fixes * Rename 'override-block-size' to 'skip-block-size-check' to free up -o * Add back 'block_rows' as an alias of 'block_size' * Add back 'behavior' in rerun and mark as deprecated * Improve merkle tree help text * Fix tests * update generated helptext * Ensure pgcrypto is present * Fix spock diff --------- Co-authored-by: Matthew Mols --- cli/genHelp.sh | 55 +- cli/scripts/ace-tests/conftest.py | 76 +- cli/scripts/ace-tests/test_api.py | 98 +- .../ace-tests/test_block_boundaries.py | 212 ++ cli/scripts/ace-tests/test_cert_auth.py | 6 - cli/scripts/ace-tests/test_composite_keys.py | 8 - cli/scripts/ace-tests/test_config.py | 2 + cli/scripts/ace-tests/test_data_types.py | 154 +- cli/scripts/ace-tests/test_fix_nulls.py | 52 +- cli/scripts/ace-tests/test_insert_only.py | 52 +- .../ace-tests/test_merkle_trees_composite.py | 699 ++++ .../ace-tests/test_merkle_trees_simple.py | 701 ++++ cli/scripts/ace-tests/test_mixed_case.py | 12 +- cli/scripts/ace-tests/test_replication.py | 5 +- cli/scripts/ace-tests/test_repset_diff.py | 95 +- cli/scripts/ace-tests/test_simple.py | 85 +- cli/scripts/ace-tests/test_simple_base.py | 7 - cli/scripts/ace-tests/test_table_filter.py | 96 +- cli/scripts/ace.py | 634 +++- cli/scripts/ace_auth.py | 249 +- cli/scripts/ace_cli.py | 1375 +++++--- cli/scripts/ace_config.py | 35 +- cli/scripts/ace_constants.py | 5 + cli/scripts/ace_core.py | 1195 ++++--- cli/scripts/ace_daemon.py | 60 +- cli/scripts/ace_data_models.py | 44 +- cli/scripts/ace_mtree.py | 2805 +++++++++++++++++ cli/scripts/ace_sql.py | 856 +++++ docs/cli_functions.md | 19 +- docs/functions/ace-mtree-build.md | 44 + docs/functions/ace-mtree-init.md | 21 + docs/functions/ace-mtree-table-diff.md | 35 + docs/functions/ace-mtree-teardown.md | 24 + docs/functions/ace-mtree-update.md | 29 + docs/functions/ace-mtree.md | 14 + docs/functions/ace-repset-diff.md | 14 +- docs/functions/ace-spock-exception-update.md | 16 +- docs/functions/ace-table-diff.md | 16 +- docs/functions/ace-table-repair.md | 2 +- docs/functions/ace-table-rerun.md | 10 +- docs/functions/ace.md | 12 +- requirements.txt | 2 +- 42 files changed, 8210 insertions(+), 1721 deletions(-) create mode 100644 cli/scripts/ace-tests/test_block_boundaries.py create mode 100644 cli/scripts/ace-tests/test_merkle_trees_composite.py create mode 100644 cli/scripts/ace-tests/test_merkle_trees_simple.py create mode 100644 cli/scripts/ace_constants.py create mode 100644 cli/scripts/ace_mtree.py create mode 100644 cli/scripts/ace_sql.py create mode 100644 docs/functions/ace-mtree-build.md create mode 100644 docs/functions/ace-mtree-init.md create mode 100644 docs/functions/ace-mtree-table-diff.md create mode 100644 docs/functions/ace-mtree-teardown.md create mode 100644 docs/functions/ace-mtree-update.md create mode 100644 docs/functions/ace-mtree.md diff --git a/cli/genHelp.sh b/cli/genHelp.sh index 10a0aedc..f24a3882 100755 --- a/cli/genHelp.sh +++ b/cli/genHelp.sh @@ -4,17 +4,26 @@ export nc=../out/posix/pgedge export output_dir=../docs modules=(ace cluster db localhost service spock um) + commands=(setup upgrade-cli) mkdir -p "$output_dir" + +get_submodules(){ + local module="$1" + + if [[ "$module" == "ace" ]]; then + echo "mtree" + fi +} parse_to_markdown(){ sed -r 's/\x1B\[[0-9;]*[mGKH]//g; /^(SYNOPSIS|POSITIONAL ARGUMENTS|DESCRIPTION|FLAGS|COMMANDS)/s/^/## /' } get_module_commands() { local module="$1" - local module_file="$output_dir/functions/$module.md" + local module_file="$output_dir/functions/${module// /-}.md" local cmds=() if [[ -f "$module_file" ]]; then local in_commands=0 @@ -57,7 +66,7 @@ module_summary() { write_help() { # Generate help for a command or module (and its subcommands) local module="$1" - $nc $module --help 2>/dev/null | parse_to_markdown > "$output_dir/functions/$module.md"; + $nc $module --help 2>/dev/null | parse_to_markdown > "$output_dir/functions/${module// /-}.md"; # Parse the generated module help file to extract subcommands (if they exist) module_commands=($(get_module_commands "$module")) @@ -69,13 +78,25 @@ write_help() { # Generate help for each command in the module for cmd in "${module_commands[@]}"; do - local fname="${module}-$(echo "$cmd" | tr ' ' '-').md" + local fname="${module// /-}-${cmd// /-}.md" echo "Generating help for module '$module', command '$cmd' -> $fname" if ! $nc $module $cmd --help 2>/dev/null | parse_to_markdown > "$output_dir/functions/$fname"; then echo "ERROR: Failed to generate help for module '$module', command '$cmd'" >&2 fi done + + # If the module has submodules, recursively generate help for them + local submodules=($(get_submodules "$module")) + if [ ${#submodules[@]} -gt 0 ]; then + echo "Found submodules for module '$module': ${submodules[*]}" + for submodule in "${submodules[@]}"; do + echo "Generating help for submodule '$submodule' in module '$module'" + write_help "$module $submodule" + done + else + echo "No submodules found for module '$module'" + fi } index() { @@ -132,6 +153,32 @@ index() { ' "$module_file" >> "$index_file" echo "" >> "$index_file" + for submodule in $(get_submodules "$module"); do + echo "### $module $submodule submodule commands" >> "$index_file" + echo "" >> "$index_file" + echo "| Command | Description |" >> "$index_file" + echo "|---------|-------------|" >> "$index_file" + + # Parse the -submodule.md file to extract commands and descriptions + submodule_file="$output_dir/functions/${module// /-}-${submodule// /-}.md" + awk -v module="$module" -v submodule="$submodule" ' + BEGIN { in_commands=0 } + /COMMAND is one of the following:/ { in_commands=1; next } + in_commands && /^[[:space:]]*$/ { exit } + in_commands && /^[[:space:]]*[^[:space:]]/ { + split($0, parts, "#") + cmd=parts[1] + gsub(/^[ \t]+|[ \t]+$/, "", cmd) + desc=parts[2] + gsub(/^[ \t]+|[ \t]+$/, "", desc) + if (cmd != "") { + printf "| [%s %s](functions/%s-%s-%s.md) | %s |\n", module, submodule, module, submodule, cmd, desc + } + } + ' "$submodule_file" >> "$index_file" + echo "" >> "$index_file" + done + done } @@ -146,7 +193,7 @@ if [ "$m" == "all" ]; then echo "Generating help for all modules..." echo "Removing existing help files..." rm -f $output_dir/functions/* - + # Loop through all modules and generate help for module in "${modules[@]}"; do write_help "$module" diff --git a/cli/scripts/ace-tests/conftest.py b/cli/scripts/ace-tests/conftest.py index 7eda905b..fbb9ce71 100644 --- a/cli/scripts/ace-tests/conftest.py +++ b/cli/scripts/ace-tests/conftest.py @@ -9,6 +9,7 @@ import test_config from test_simple_base import TestSimpleBase from test_simple import TestSimple +from test_merkle_trees_simple import TestMerkleTreesSimple # Set up paths os.environ["PGEDGE_HOME"] = test_config.PGEDGE_HOME @@ -49,7 +50,12 @@ def set_run_dir(): @pytest.fixture(scope="session") def cli(): - return load_mod("ace_cli") + return load_mod("ace_cli").AceCLI() + + +@pytest.fixture(scope="session") +def mtree_cli(cli): + return cli.mtree() @pytest.fixture(scope="session") @@ -226,13 +232,24 @@ def prepare_spock(node): sleep(5) +def pytest_addoption(parser): + parser.addoption( + "--skip-cleanup", action="store_true", help="Skip DB cleanup fixture" + ) + + @pytest.fixture(scope="session", autouse=True) -def cleanup_databases(nodes): +def cleanup_databases(request, nodes): """Cleanup all databases after running tests""" # Yield to let the tests run first yield + skip = request.config.getoption("--skip-cleanup") + + if skip: + pytest.skip("Skipping DB cleanup") + # Cleanup code that runs after all tests complete drop_customers_sql = "DROP TABLE IF EXISTS customers CASCADE;" @@ -335,22 +352,43 @@ def pytest_configure(config): def pytest_collection_modifyitems(items): - """Skip tests marked as abstract_base if they are in the base class.""" + """ + Skips tests from TestSimpleBase as they should not be run directly. + """ for item in items: - if item.get_closest_marker("abstract_base"): - # Skip only if the test is in TestSimpleBase class directly - # or if the test method is not overridden in the child class - if item.cls and ( - ( - item.cls.__name__ == "TestSimpleBase" - and ( - issubclass(item.cls, TestSimpleBase) - and item.function.__qualname__.startswith("TestSimpleBase.") - ) - ) - or ( - issubclass(item.cls, TestSimple) - and item.function.__qualname__.startswith("TestSimple.") + if ( + item.cls + and issubclass(item.cls, TestSimpleBase) + and item.function.__qualname__.startswith("TestSimpleBase.") + ): + item.add_marker( + pytest.mark.skip( + reason="TestSimpleBase tests are not meant to be run directly" ) - ): - item.add_marker(pytest.mark.skip(reason="Abstract base class")) + ) + + +def pytest_runtest_setup(item): + """ + Skip parent class tests if a child class test is also in the run. + """ + if not item.get_closest_marker("abstract_base"): + return + + if item.cls is TestMerkleTreesSimple: + is_child_running = any( + i.cls + and issubclass(i.cls, TestMerkleTreesSimple) + and i.cls is not TestMerkleTreesSimple + for i in item.session.items + ) + if is_child_running: + pytest.skip("Skipping parent class") + + if item.cls is TestSimple: + is_child_running = any( + i.cls and issubclass(i.cls, TestSimple) and i.cls is not TestSimple + for i in item.session.items + ) + if is_child_running: + pytest.skip("Skipping parent class") diff --git a/cli/scripts/ace-tests/test_api.py b/cli/scripts/ace-tests/test_api.py index cccb570c..64d057cd 100644 --- a/cli/scripts/ace-tests/test_api.py +++ b/cli/scripts/ace-tests/test_api.py @@ -8,6 +8,7 @@ from test_simple_base import TestSimpleBase +# @pytest.mark.skip(reason="Skipping API tests") @pytest.mark.usefixtures("prepare_databases") class TestAPI(TestSimpleBase): @pytest.fixture(scope="class", autouse=True) @@ -52,7 +53,7 @@ def test_simple_table_diff(self, cli, capsys, ace_conf, table_name): "cluster_name": "eqn-t9da", "table_name": table_name, "dbname": "demo", - "block_rows": 10000, + "block_size": 10000, "max_cpu_ratio": 0.6, "output": "json", "nodes": "all", @@ -137,7 +138,7 @@ def test_table_diff_with_differences( "cluster_name": "eqn-t9da", "table_name": table_name, "dbname": "demo", - "block_rows": 10000, + "block_size": 10000, "max_cpu_ratio": 0.6, "output": "json", "nodes": "all", @@ -307,7 +308,7 @@ def test_table_rerun_temptable( "cluster_name": "eqn-t9da", "table_name": table_name, "dbname": "demo", - "block_rows": 10000, + "block_size": 10000, "max_cpu_ratio": 0.6, "output": "json", "nodes": "all", @@ -358,7 +359,6 @@ def test_table_rerun_temptable( "diff_file": diff_file_path.path, "table_name": table_name, "dbname": "demo", - "behavior": "hostdb", "quiet": False, } @@ -417,93 +417,11 @@ def test_table_rerun_temptable( "-modified" ), f"Modified row {diff[key_column]} doesn't have expected suffix" - except Exception as e: - pytest.fail(f"Test failed: {str(e)}") - - @pytest.mark.parametrize("table_name", ["public.customers"]) - def test_table_rerun_multiprocessing( - self, - cli, - capsys, - ace_conf, - table_name, - diff_file_path, - ): - """Test table rerun API (multiprocessing mode) on cluster eqn-t9da""" - max_retries = 30 - retry_count = 0 - task_completed = False - cert_config = self._get_cert_config(ace_conf) - - try: - rerun_payload = { - "cluster_name": "eqn-t9da", - "diff_file": diff_file_path.path, - "table_name": table_name, - "dbname": "demo", - "behavior": "multiprocessing", - "quiet": False, - } - - rerun_response = requests.post( - f"{self._get_api_base_url()}/table-rerun", - json=rerun_payload, - **cert_config, - ) - - assert rerun_response.status_code == 200 - rerun_task_id = rerun_response.json()["task_id"] - - # Wait for rerun to complete - retry_count = 0 - task_completed = False - - while retry_count < max_retries and not task_completed: - status_response = requests.get( - f"{self._get_api_base_url()}/task-status", - params={"task_id": rerun_task_id}, - **cert_config, - ) - - assert status_response.status_code == 200 - status_data = status_response.json() - - if status_data["task_status"] == "COMPLETED": - task_completed = True - elif status_data["task_status"] == "FAILED": - error_msg = status_data.get("error_message", "Unknown error") - pytest.fail(f"Rerun task failed: {error_msg}") - else: - time.sleep(1) - retry_count += 1 - - assert task_completed, "Rerun task did not complete within timeout period" - - # Verify the diff file contains 50 differences - with open(diff_file_path.path, "r") as f: - diff_data = json.load(f) - - assert ( - len(diff_data["diffs"]["n1/n2"]["n2"]) == 50 - ), "Expected 50 differences" - - # Verify the differences are correctly reported - for diff in diff_data["diffs"]["n1/n2"]["n2"]: - assert diff["first_name"].endswith( - "-modified" - ), f"Modified row {diff['index']} doesn't have expected suffix" - - # Verify the control rows are not modified - for diff in diff_data["diffs"]["n1/n2"]["n1"]: - assert not diff["first_name"].endswith( - "-modified" - ), f"Control row {diff['index']} shouldn't have modification suffix" - # Repair to restore state - cli.table_repair_cli( - "eqn-t9da", - table_name, - diff_file_path.path, + cli.table_repair( + cluster_name="eqn-t9da", + diff_file=diff_file_path.path, + table_name=table_name, source_of_truth="n1", ) diff --git a/cli/scripts/ace-tests/test_block_boundaries.py b/cli/scripts/ace-tests/test_block_boundaries.py new file mode 100644 index 00000000..807b5421 --- /dev/null +++ b/cli/scripts/ace-tests/test_block_boundaries.py @@ -0,0 +1,212 @@ +import logging +import random +import pytest +import psycopg +from psycopg import sql +import re +import json + +import test_config + + +@pytest.mark.usefixtures("prepare_databases") +class TestBlockBoundaries: + """Tests for table-diff block boundaries handling""" + + @pytest.fixture(scope="class") + def setup_test_scenario(self, nodes): + """Setup fixture to create different scenarios for block boundary testing""" + try: + for node in nodes: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + customers1_sql = """ + CREATE TABLE customers1 ( + index INTEGER PRIMARY KEY NOT NULL, + customer_id TEXT NOT NULL, + first_name TEXT NOT NULL, + last_name TEXT NOT NULL, + company TEXT NOT NULL, + city TEXT NOT NULL, + country TEXT NOT NULL, + phone_1 TEXT NOT NULL, + phone_2 TEXT NOT NULL, + email TEXT NOT NULL, + subscription_date TIMESTAMP WITHOUT TIME ZONE NOT NULL, + website TEXT NOT NULL + ); + """ + + customers2_sql = """ + CREATE TABLE customers2 ( + index INTEGER PRIMARY KEY NOT NULL, + customer_id TEXT NOT NULL, + first_name TEXT NOT NULL, + last_name TEXT NOT NULL, + company TEXT NOT NULL, + city TEXT NOT NULL, + country TEXT NOT NULL, + phone_1 TEXT NOT NULL, + phone_2 TEXT NOT NULL, + email TEXT NOT NULL, + subscription_date TIMESTAMP WITHOUT TIME ZONE NOT NULL, + website TEXT NOT NULL + ); + """ + + cur.execute(customers1_sql) + cur.execute(customers2_sql) + + with open(test_config.CUSTOMERS1_CSV, "r") as f: + with cur.copy( + "COPY customers1 FROM STDIN CSV HEADER DELIMITER ','" + ) as copy: + copy.write(f.read()) + + with open(test_config.CUSTOMERS2_CSV, "r") as f: + with cur.copy( + "COPY customers2 FROM STDIN CSV HEADER DELIMITER ','" + ) as copy: + copy.write(f.read()) + + cur.execute( + "SELECT spock.repset_add_table('test_repset', 'customers1')" + ) + cur.execute( + "SELECT spock.repset_add_table('test_repset', 'customers2')" + ) + + conn.commit() + cur.close() + conn.close() + + print(f"Created customers1 and customers2 tables on {node}") + + yield + + # Cleanup + for node in nodes: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + + cur.execute("DROP TABLE customers1 cascade") + cur.execute("DROP TABLE customers2 cascade") + conn.commit() + cur.close() + conn.close() + + print(f"Dropped customers1 and customers2 tables on {node}") + + except Exception as e: + pytest.fail(f"Test failed: {str(e)}") + + @pytest.mark.usefixtures("setup_test_scenario") + @pytest.mark.parametrize( + "table, row_count, block_size", + [ + ("customers", 10000, 10), + ("customers", 10000, 100), + ("customers", 10000, 1000), + ("customers", 10000, 5000), + ("customers1", 10500, 10), + ("customers1", 10500, 100), + ("customers1", 10500, 1000), + ("customers1", 10500, 5000), + ("customers1", 10500, 10000), + ("customers2", 1000000, 1000), + ("customers2", 1000000, 10000), + ("customers2", 1000000, 100000), + ("customers2", 1000000, 500000), + ], + ) + @pytest.mark.parametrize("max_sample_size", [50]) + def test_block_boundaries( + self, caplog, cli, capsys, table, row_count, block_size, max_sample_size + ): + """Test that table-diff correctly identifies differences at block boundaries""" + caplog.set_level(logging.INFO) + try: + # Better to randomise this instead of using hardcoded values + multiplier = row_count // block_size + rand_boundaries = list( + set( + [ + block_size * random.randint(1, multiplier) # nosec: B311 + for _ in range(1, multiplier + 1) + ] + ) + ) + block_boundaries = [b - 1 for b in rand_boundaries] + [ + b + 1 for b in rand_boundaries + ] + + block_boundaries.sort() + + if len(block_boundaries) > max_sample_size: + block_boundaries = random.sample(block_boundaries, max_sample_size) + + block_boundaries = block_boundaries + [row_count] + + block_boundaries = list(filter(lambda x: x <= row_count, block_boundaries)) + + logging.getLogger().info(f"Block boundaries: {block_boundaries}") + + conn = psycopg.connect(host="n2", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + + for boundary in block_boundaries: + cur.execute( + sql.SQL( + """ + UPDATE {schema}.{table} + SET first_name = 'Modified' + WHERE index = {boundary} + """ + ).format( + schema=sql.Identifier("public"), + table=sql.Identifier(table), + boundary=boundary, + ) + ) + + conn.commit() + cur.close() + conn.close() + + # Run table-diff with block size of 1000 + cli.table_diff( + "eqn-t9da", + f"public.{table}", + block_size=block_size, + skip_block_size_check=True, + ) + + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file = match.group(1) + + with open(diff_file, "r") as f: + diff_data = json.load(f) + + assert len(diff_data["diffs"]["n1/n2"]["n2"]) == len( + block_boundaries + ), f"Expected {len(block_boundaries)} diffs," + f" got {len(diff_data['diffs']['n1/n2']['n2'])}" + + for diff in diff_data["diffs"]["n1/n2"]["n2"]: + assert ( + diff["first_name"] == "Modified" + ), "Expected first_name to be Modified" + + # Repair everything now + cli.table_repair( + "eqn-t9da", f"public.{table}", diff_file, source_of_truth="n1" + ) + + except Exception as e: + pytest.fail(f"Test failed: {str(e)}") diff --git a/cli/scripts/ace-tests/test_cert_auth.py b/cli/scripts/ace-tests/test_cert_auth.py index 9d8cb32a..40d519da 100644 --- a/cli/scripts/ace-tests/test_cert_auth.py +++ b/cli/scripts/ace-tests/test_cert_auth.py @@ -60,9 +60,3 @@ def test_table_rerun_temptable( return super().test_table_rerun_temptable( cli, capsys, ace_conf, table_name, key_column, diff_file_path ) - - @pytest.mark.parametrize("table_name", ["public.customers"]) - def test_table_rerun_multiprocessing(self, cli, capsys, table_name, diff_file_path): - return super().test_table_rerun_multiprocessing( - cli, capsys, table_name, diff_file_path - ) diff --git a/cli/scripts/ace-tests/test_composite_keys.py b/cli/scripts/ace-tests/test_composite_keys.py index 8a2ba8bb..68503114 100644 --- a/cli/scripts/ace-tests/test_composite_keys.py +++ b/cli/scripts/ace-tests/test_composite_keys.py @@ -112,11 +112,3 @@ def test_table_rerun_temptable( return super().test_table_rerun_temptable( cli, capsys, ace_conf, table_name, key_column, diff_file_path ) - - @pytest.mark.parametrize("table_name", ["public.customers"]) - def test_table_rerun_multiprocessing( - self, cli, capsys, table_name, diff_file_path - ): - return super().test_table_rerun_multiprocessing( - cli, capsys, table_name, diff_file_path - ) diff --git a/cli/scripts/ace-tests/test_config.py b/cli/scripts/ace-tests/test_config.py index 1b50b2d3..257db788 100644 --- a/cli/scripts/ace-tests/test_config.py +++ b/cli/scripts/ace-tests/test_config.py @@ -4,3 +4,5 @@ TESTS_DIR = "/home/test/dev/cli/cli/scripts/ace-tests" CUSTOMERS_SQL = os.path.join(TESTS_DIR, "customers.sql") CUSTOMERS_CSV = os.path.join(TESTS_DIR, "customers.csv") +CUSTOMERS1_CSV = os.path.join(TESTS_DIR, "customers1.csv") +CUSTOMERS2_CSV = os.path.join(TESTS_DIR, "customers2.csv") diff --git a/cli/scripts/ace-tests/test_data_types.py b/cli/scripts/ace-tests/test_data_types.py index 6b3c2b83..950bf2dd 100644 --- a/cli/scripts/ace-tests/test_data_types.py +++ b/cli/scripts/ace-tests/test_data_types.py @@ -164,7 +164,10 @@ def test_table_diff_with_differences( conn.close() # Execute table diff and verify results - cli.table_diff_cli("eqn-t9da", table_name) + cli.table_diff( + cluster_name="eqn-t9da", + table_name=table_name, + ) # Capture and verify output captured = capsys.readouterr() @@ -287,7 +290,10 @@ def test_table_rerun_temptable( pytest.fail(f"Failed to introduce diffs on n2: {str(e)}") # Run table-diff to get the diff file - cli.table_diff_cli("eqn-t9da", table_name) + cli.table_diff( + cluster_name="eqn-t9da", + table_name=table_name, + ) captured = capsys.readouterr() clean_output = re.sub( r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out @@ -297,7 +303,12 @@ def test_table_rerun_temptable( path = match.group(1) # Run table-rerun with hostdb behavior - cli.table_rerun_cli("eqn-t9da", path, table_name, "demo", False, "hostdb") + cli.table_rerun( + cluster_name="eqn-t9da", + diff_file=path, + table_name=table_name, + dbname="demo", + ) captured = capsys.readouterr() clean_output = re.sub( @@ -349,120 +360,6 @@ def test_table_rerun_temptable( "rerun-modified", ), f"Modified row {diff['id']} doesn't have expected value" - @pytest.mark.parametrize("table_name", ["public.datatypes_test"]) - @pytest.mark.parametrize( - "column_name,test_value", - [ - ("int_col", "5678"), - ("float_col", "76.543"), - ("array_col", "ARRAY[44, 55, 66]"), - ("json_col", '\'{"multiproc": "modified"}\''), - ("bytea_col", "decode('98765432', 'hex')"), - ("point_col", "point(77.7, 77.7)"), - ("text_col", "'multiproc-modified'"), - ("text_array_col", "ARRAY['multiproc', 'modified', 'array']"), - ], - ) - def test_table_rerun_multiprocessing( - self, - cli, - capsys, - table_name, - column_name, - test_value, - diff_file_path, - ): - """Test table rerun multiprocessing with various data types""" - - try: - conn = psycopg.connect(host="n2", dbname="demo", user="admin") - cur = conn.cursor() - cur.execute("SELECT spock.repair_mode(true)") - cur.execute( - f""" - UPDATE datatypes_test - SET {column_name} = {test_value} - WHERE id IN ( - SELECT id - FROM datatypes_test - ) - """ - ) - conn.commit() - cur.close() - conn.close() - except Exception as e: - pytest.fail(f"Failed to introduce diffs on n2: {str(e)}") - - # Run table-diff to get the diff file - cli.table_diff_cli("eqn-t9da", table_name) - captured = capsys.readouterr() - clean_output = re.sub( - r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out - ) - match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) - assert match, "Diff file path not found in output" - path = match.group(1) - - cli.table_rerun_cli( - "eqn-t9da", - path, - table_name, - "demo", - False, - "multiprocessing", - ) - - captured = capsys.readouterr() - clean_output = re.sub( - r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out - ) - match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) - assert match, "Diff file path not found in output" - diff_file_path.path = match.group(1) - - # Verify the diffs - with open(diff_file_path.path, "r") as f: - diff_data = json.load(f) - - assert ( - len(diff_data["diffs"]["n1/n2"]["n2"]) == 3 - ), f"Expected 3 differences, found {len(diff_data['diffs']['n1/n2']['n2'])}" - - # Verify the differences are correctly reported - for diff in diff_data["diffs"]["n1/n2"]["n2"]: - if column_name == "json_col": - assert ( - diff[column_name].get("multiproc") == "modified" - ), f"Modified row {diff['id']} doesn't have expected JSON value" - elif column_name == "array_col": - assert diff[column_name] == [ - 44, - 55, - 66, - ], f"Modified row {diff['id']} doesn't have expected array value" - elif column_name == "text_array_col": - assert diff[column_name] == [ - "multiproc", - "modified", - "array", - ], f"Modified row {diff['id']} doesn't have expected text array value" - elif column_name == "point_col": - assert ( - diff[column_name] == "(77.7,77.7)" - ), f"Modified row {diff['id']} doesn't have expected point value" - elif column_name == "bytea_col": - assert ( - diff[column_name] == "98765432" - ), f"Modified row {diff['id']} doesn't have expected bytea value" - else: - assert str(diff[column_name]) in ( - test_value.strip("'"), - "5678", - "76.543", - "multiproc-modified", - ), f"Modified row {diff['id']} doesn't have expected value" - @pytest.mark.parametrize("table_name", ["public.datatypes_test"]) @pytest.mark.parametrize( "column_name,test_value,expected_value", @@ -471,13 +368,13 @@ def test_table_rerun_multiprocessing( ("float_col", "123.456", 123.456), ("array_col", "ARRAY[99, 98, 97]", [99, 98, 97]), ("json_col", '\'{"test": "modified"}\'', {"test": "modified"}), - ("bytea_col", "decode('FEEDFACE', 'hex')", b'\xfe\xed\xfa\xce'), + ("bytea_col", "decode('FEEDFACE', 'hex')", b"\xfe\xed\xfa\xce"), ("point_col", "point(99.9, 99.9)", "(99.9,99.9)"), ("text_col", "'modified-text'", "modified-text"), ( "text_array_col", "ARRAY['modified', 'text', 'array']", - ["modified", "text", "array"] + ["modified", "text", "array"], ), ], ) @@ -512,23 +409,24 @@ def test_table_repair_datatypes( conn.close() # Run table-diff to get the diff file - cli.table_diff_cli("eqn-t9da", table_name) + cli.table_diff( + cluster_name="eqn-t9da", + table_name=table_name, + ) captured = capsys.readouterr() clean_output = re.sub( - r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", - "", - captured.out + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out ) match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) assert match, "Diff file path not found in output" diff_file_path.path = match.group(1) # Run table-repair using n2 as source of truth - cli.table_repair_cli( - "eqn-t9da", - table_name, - diff_file_path.path, - source_of_truth="n2" + cli.table_repair( + cluster_name="eqn-t9da", + table_name=table_name, + diff_file=diff_file_path.path, + source_of_truth="n2", ) # Verify the repair worked by checking values on n1 diff --git a/cli/scripts/ace-tests/test_fix_nulls.py b/cli/scripts/ace-tests/test_fix_nulls.py index 293b13f0..aef26cba 100644 --- a/cli/scripts/ace-tests/test_fix_nulls.py +++ b/cli/scripts/ace-tests/test_fix_nulls.py @@ -307,7 +307,10 @@ def setup_datatype_nulls(self, nodes): def test_simple_nulls(self, cli, capsys): """Test fix-nulls with simple primary key""" # First run table-diff to get the diff file - cli.table_diff_cli("eqn-t9da", "public.simple_nulls") + cli.table_diff( + cluster_name="eqn-t9da", + table_name="public.simple_nulls", + ) captured = capsys.readouterr() output = captured.out @@ -318,10 +321,10 @@ def test_simple_nulls(self, cli, capsys): diff_file_path = match.group(1) # Run table-repair with fix-nulls - cli.table_repair_cli( - "eqn-t9da", - "public.simple_nulls", - diff_file_path, + cli.table_repair( + cluster_name="eqn-t9da", + table_name="public.simple_nulls", + diff_file=diff_file_path, fix_nulls=True, ) @@ -344,7 +347,10 @@ def test_simple_nulls(self, cli, capsys): def test_composite_nulls(self, cli, capsys): """Test fix-nulls with composite primary key""" # First run table-diff to get the diff file - cli.table_diff_cli("eqn-t9da", "public.composite_nulls") + cli.table_diff( + cluster_name="eqn-t9da", + table_name="public.composite_nulls", + ) captured = capsys.readouterr() output = captured.out @@ -355,10 +361,10 @@ def test_composite_nulls(self, cli, capsys): diff_file_path = match.group(1) # Run table-repair with fix-nulls - cli.table_repair_cli( - "eqn-t9da", - "public.composite_nulls", - diff_file_path, + cli.table_repair( + cluster_name="eqn-t9da", + table_name="public.composite_nulls", + diff_file=diff_file_path, fix_nulls=True, ) @@ -379,7 +385,10 @@ def test_composite_nulls(self, cli, capsys): def test_mixed_case_nulls(self, cli, capsys): """Test fix-nulls with mixed case table and column names""" # First run table-diff to get the diff file - cli.table_diff_cli("eqn-t9da", "public.MixedCaseNulls") + cli.table_diff( + cluster_name="eqn-t9da", + table_name="public.MixedCaseNulls", + ) captured = capsys.readouterr() output = captured.out @@ -390,10 +399,10 @@ def test_mixed_case_nulls(self, cli, capsys): diff_file_path = match.group(1) # Run table-repair with fix-nulls - cli.table_repair_cli( - "eqn-t9da", - "public.MixedCaseNulls", - diff_file_path, + cli.table_repair( + cluster_name="eqn-t9da", + table_name="public.MixedCaseNulls", + diff_file=diff_file_path, fix_nulls=True, ) @@ -414,7 +423,10 @@ def test_mixed_case_nulls(self, cli, capsys): def test_datatype_nulls(self, cli, capsys): """Test fix-nulls with various datatypes""" # First run table-diff to get the diff file - cli.table_diff_cli("eqn-t9da", "public.datatype_nulls") + cli.table_diff( + cluster_name="eqn-t9da", + table_name="public.datatype_nulls", + ) captured = capsys.readouterr() output = captured.out @@ -425,10 +437,10 @@ def test_datatype_nulls(self, cli, capsys): diff_file_path = match.group(1) # Run table-repair with fix-nulls - cli.table_repair_cli( - "eqn-t9da", - "public.datatype_nulls", - diff_file_path, + cli.table_repair( + cluster_name="eqn-t9da", + table_name="public.datatype_nulls", + diff_file=diff_file_path, fix_nulls=True, ) diff --git a/cli/scripts/ace-tests/test_insert_only.py b/cli/scripts/ace-tests/test_insert_only.py index 1db9641e..994c98f7 100644 --- a/cli/scripts/ace-tests/test_insert_only.py +++ b/cli/scripts/ace-tests/test_insert_only.py @@ -88,7 +88,10 @@ def test_insert_only_repair(self, cli, capsys): modifying existing ones """ try: - cli.table_diff_cli("eqn-t9da", "public.insert_only_test") + cli.table_diff( + cluster_name="eqn-t9da", + table_name="public.insert_only_test", + ) captured = capsys.readouterr() clean_output = re.sub( r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out @@ -97,10 +100,10 @@ def test_insert_only_repair(self, cli, capsys): assert match, "Diff file path not found in output" diff_file = match.group(1) - cli.table_repair_cli( - "eqn-t9da", - "public.insert_only_test", - diff_file, + cli.table_repair( + cluster_name="eqn-t9da", + table_name="public.insert_only_test", + diff_file=diff_file, source_of_truth="n1", insert_only=True, ) @@ -168,7 +171,10 @@ def test_insert_only_with_deletes(self, cli, capsys): cur.close() conn.close() - cli.table_diff_cli("eqn-t9da", "public.insert_only_test") + cli.table_diff( + cluster_name="eqn-t9da", + table_name="public.insert_only_test", + ) captured = capsys.readouterr() clean_output = re.sub( r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out @@ -177,10 +183,10 @@ def test_insert_only_with_deletes(self, cli, capsys): assert match, "Diff file path not found in output" diff_file = match.group(1) - cli.table_repair_cli( - "eqn-t9da", - "public.insert_only_test", - diff_file, + cli.table_repair( + cluster_name="eqn-t9da", + table_name="public.insert_only_test", + diff_file=diff_file, source_of_truth="n1", insert_only=True, ) @@ -296,7 +302,10 @@ def test_bidirectional_insert(self, cli, capsys): """Test that bidirectional insert propagates missing rows in both directions""" try: print("Running table-diff on bidirectional_test") - cli.table_diff_cli("eqn-t9da", "public.bidirectional_test") + cli.table_diff( + cluster_name="eqn-t9da", + table_name="public.bidirectional_test", + ) captured = capsys.readouterr() clean_output = re.sub( r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out @@ -306,10 +315,10 @@ def test_bidirectional_insert(self, cli, capsys): diff_file = match.group(1) print("Running table-repair on bidirectional_test") - cli.table_repair_cli( - "eqn-t9da", - "public.bidirectional_test", - diff_file, + cli.table_repair( + cluster_name="eqn-t9da", + table_name="public.bidirectional_test", + diff_file=diff_file, insert_only=True, bidirectional=True, ) @@ -434,7 +443,10 @@ def test_bidirectional_insert_with_updates(self, cli, capsys): cur.close() conn.close() - cli.table_diff_cli("eqn-t9da", "public.bidirectional_test") + cli.table_diff( + cluster_name="eqn-t9da", + table_name="public.bidirectional_test", + ) captured = capsys.readouterr() clean_output = re.sub( r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out @@ -443,10 +455,10 @@ def test_bidirectional_insert_with_updates(self, cli, capsys): assert match, "Diff file path not found in output" diff_file = match.group(1) - cli.table_repair_cli( - "eqn-t9da", - "public.bidirectional_test", - diff_file, + cli.table_repair( + cluster_name="eqn-t9da", + table_name="public.bidirectional_test", + diff_file=diff_file, insert_only=True, bidirectional=True, ) diff --git a/cli/scripts/ace-tests/test_merkle_trees_composite.py b/cli/scripts/ace-tests/test_merkle_trees_composite.py new file mode 100644 index 00000000..a3e5ee8a --- /dev/null +++ b/cli/scripts/ace-tests/test_merkle_trees_composite.py @@ -0,0 +1,699 @@ +import random +import pytest +from psycopg import sql +import psycopg +import re +import json +from faker import Faker +from test_merkle_trees_simple import TestMerkleTreesSimple + +from psycopg.types.composite import register_composite, CompositeInfo + + +@pytest.mark.usefixtures("prepare_databases") +class TestMerkleTreesComposite(TestMerkleTreesSimple): + """Tests for merkle tree operations with composite keys""" + + @pytest.fixture(scope="class", autouse=True) + def setup_composite_keys(self, nodes, setup_test_scenario): + """Setup fixture to create different scenarios for block boundary testing""" + + try: + for node in nodes: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + sql.SQL("ALTER TABLE customers DROP CONSTRAINT customers_pkey") + ) + cur.execute( + sql.SQL( + "ALTER TABLE customers ADD PRIMARY KEY (index, customer_id)" + ) + ) + + cur.execute( + sql.SQL("ALTER TABLE customers2 DROP CONSTRAINT customers2_pkey") + ) + cur.execute( + sql.SQL( + "ALTER TABLE customers2 ADD PRIMARY KEY (index, customer_id)" + ) + ) + + conn.commit() + cur.close() + conn.close() + + print(f"Created customers2 table on {node}") + + yield + + # Cleanup + for node in nodes: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + sql.SQL("ALTER TABLE customers DROP CONSTRAINT customers_pkey") + ) + cur.execute(sql.SQL("ALTER TABLE customers ADD PRIMARY KEY (index)")) + conn.commit() + cur.close() + conn.close() + + print(f"Reverted original primary key for customers on {node}") + + except Exception as e: + pytest.fail(f"Test failed: {str(e)}") + + @pytest.mark.parametrize("cluster_name", ["eqn-t9da"]) + def test_merkle_tree_init(self, mtree_cli, capsys, cluster_name, nodes): + return super().test_merkle_tree_init(mtree_cli, capsys, cluster_name, nodes) + + @pytest.mark.parametrize("table", ["public.customers", "public.customers2"]) + def test_merkle_tree_setup(self, mtree_cli, capsys, table, nodes): + return super().test_merkle_tree_setup(mtree_cli, capsys, table, nodes) + + @pytest.mark.parametrize( + "table, diff_count", [("public.customers", 75), ("public.customers2", 1000)] + ) + def test_simple_diff(self, cli, mtree_cli, capsys, table, diff_count): + return super().test_simple_diff(cli, mtree_cli, capsys, table, diff_count) + + @pytest.mark.parametrize("table", ["public.customers", "public.customers2"]) + def test_boundaries(self, cli, mtree_cli, capsys, table): + """Test block boundaries""" + + l_schema, l_table = table.split(".") + + mtree_table = "" + + if table == "public.customers": + mtree_table = "ace_mtree_public_customers" + else: + mtree_table = "ace_mtree_public_customers2" + + conn = psycopg.connect(host="n2", dbname="demo", user="admin") + info = CompositeInfo.fetch(conn, f"{l_schema}_{l_table}_key_type") + register_composite(info, conn, factory=lambda *args: tuple(args)) + + cur = conn.cursor() + + select_sql = """ + SELECT range_start, range_end + FROM {schema}.{mtree_table} WHERE node_level = 0 + ORDER BY node_position + """ + + # nosemgrep + cur.execute( + sql.SQL(select_sql).format( + mtree_table=sql.Identifier(mtree_table), + schema=sql.Identifier(l_schema), + ) + ) + + ranges = cur.fetchall() + + ranges_to_modify = ( + [ranges[0][0][0]] + + [ + random.choice(r)[0] # nosec: B311 + for r in random.sample(ranges, k=int(0.5 * len(ranges))) # nosec: B311 + ] + + [ranges[-1][1][0]] + ) + + ranges_to_modify = list(set(ranges_to_modify)) + ranges_to_modify.sort() + + expected_diff_count = len(ranges_to_modify) + + cur.execute("SELECT spock.repair_mode(true)") + for range in ranges_to_modify: + # nosemgrep + cur.execute( + sql.SQL( + """ + UPDATE {schema}.{table} + SET first_name = 'Modified' + WHERE index = {range} + """ + ).format( + schema=sql.Identifier(l_schema), + table=sql.Identifier(l_table), + range=range, + ) + ) + + conn.commit() + cur.close() + conn.close() + + mtree_cli.table_diff("eqn-t9da", table_name=table) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file = match.group(1) + + with open(diff_file, "r") as f: + diff_data = json.load(f) + + assert len(diff_data["diffs"]["n1/n2"]["n2"]) == expected_diff_count + + for diff in diff_data["diffs"]["n1/n2"]["n2"]: + assert diff["first_name"] == "Modified" + + # Repair everything now + cli.table_repair( + cluster_name="eqn-t9da", + table_name=table, + diff_file=diff_file, + source_of_truth="n1", + ) + + @pytest.mark.parametrize("table", ["public.customers"]) + def test_split_ranges(self, cli, mtree_cli, capsys, table): + """ + Testing range splits automatically tests merges as well. + """ + + l_schema, l_table = table.split(".") + mtree_table = "ace_mtree_public_customers" + + # We'll first delete keys en masse. Then get the new ranges, and finally + # insert records to trigger the split. + conn = psycopg.connect(host="n2", dbname="demo", user="admin") + info = CompositeInfo.fetch(conn, f"{l_schema}_{l_table}_key_type") + register_composite(info, conn, factory=lambda *args: tuple(args)) + cur = conn.cursor() + + cur.execute(sql.SQL("SELECT spock.repair_mode(true)")) + # nosemgrep + cur.execute( + sql.SQL("DELETE FROM {schema}.{table} WHERE index <= 4000").format( + schema=sql.Identifier(l_schema), table=sql.Identifier(l_table) + ) + ) + + conn.commit() + + # We're going to deliberately rebalance the tree to make way for the splits. + mtree_cli.update( + "eqn-t9da", table_name=table, nodes="n2", rebalance=True + ) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + print(clean_output) + + assert "successfully updated" in clean_output.lower() + + # check if the merge from the rebalance has happened + + # nosemgrep + cur.execute( + sql.SQL( + """ + SELECT range_end + FROM {schema}.{mtree_table} + where node_level = 0 + order by node_position + limit 1 + """ + ).format( + mtree_table=sql.Identifier(mtree_table), + schema=sql.Identifier(l_schema), + ) + ) + + range_end = cur.fetchone()[0][0] + assert range_end > 4000 + + # Now we'll insert records to trigger the split. + self.insert_records(conn, l_schema, l_table, ["index", "customer_id"], 2000) + + mtree_cli.update( + "eqn-t9da", table_name=table, nodes="n2", rebalance=True + ) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + print(clean_output) + assert "successfully updated" in clean_output.lower() + + # check if the split has happened + # nosemgrep + cur.execute( + sql.SQL( + """ + SELECT range_end + FROM {schema}.{mtree_table} + where node_level = 0 + order by node_position + limit 1 + """ + ).format( + schema=sql.Identifier(l_schema), + mtree_table=sql.Identifier(mtree_table), + ) + ) + + range_end = cur.fetchone()[0][0] + assert range_end <= 2000 + + mtree_cli.table_diff("eqn-t9da", table_name=table) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + diff_file = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert diff_file, "Diff file path not found in output" + diff_file = diff_file.group(1) + + cli.table_repair( + cluster_name="eqn-t9da", + table_name=table, + diff_file=diff_file, + source_of_truth="n1", + ) + cur.close() + conn.close() + + @pytest.mark.parametrize("table", ["public.customers2"]) + def test_merges(self, cli, mtree_cli, capsys, table): + """Test merges""" + + l_schema, l_table = table.split(".") + mtree_table = "ace_mtree_public_customers2" + + conn = psycopg.connect(host="n2", dbname="demo", user="admin") + info = CompositeInfo.fetch(conn, f"{l_schema}_{l_table}_key_type") + register_composite(info, conn, factory=lambda *args: tuple(args)) + cur = conn.cursor() + cur.execute(sql.SQL("SELECT spock.repair_mode(true)")) + + # delete 100k-200k and >=900k + # nosemgrep + cur.execute( + sql.SQL( + """ + DELETE FROM {schema}.{table} + WHERE (index >= 100000 AND index <= 200000) OR index >= 900000 + """ + ).format(schema=sql.Identifier(l_schema), table=sql.Identifier(l_table)) + ) + + conn.commit() + + mtree_cli.table_diff("eqn-t9da", table_name=table, rebalance=True) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + diff_file = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert diff_file, "Diff file path not found in output" + + diff_file = diff_file.group(1) + + # nosemgrep + cur.execute( + sql.SQL( + """ + WITH block_data AS + ( + SELECT node_position, range_start, range_end + FROM {schema}.{mtree_table} + WHERE node_level = 0 + ) + SELECT + COUNT(t.*) AS cnt + FROM block_data b + LEFT JOIN {schema}.{table} t + ON ROW({pkey_cols}) >= b.range_start + AND (ROW({pkey_cols}) <= b.range_end OR b.range_end IS NULL) + GROUP BY + b.node_position, + b.range_start, + b.range_end + ORDER BY b.node_position; + """ + ).format( + schema=sql.Identifier(l_schema), + table=sql.Identifier(l_table), + mtree_table=sql.Identifier(mtree_table), + pkey_cols=sql.SQL(",").join( + [sql.Identifier(col) for col in ["index", "customer_id"]] + ), + ) + ) + + counts = cur.fetchall() + assert all(count[0] > 0 for count in counts) + + cli.table_repair( + cluster_name="eqn-t9da", + table_name=table, + diff_file=diff_file, + source_of_truth="n1", + ) + cur.close() + conn.close() + + @pytest.mark.parametrize("table", ["public.customers"]) + def test_non_contiguous_delete(self, cli, mtree_cli, capsys, table): + """Test non-contiguous deletes only mark affected blocks as dirty""" + + l_schema, l_table = table.split(".") + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + + # Ideally, random ranges should be used here, but the sampling might + # unnecessarily add complexity (uniform sampling within a skewed distribution). + # So, we'll just use hardcoded ranges for now. + # nosemgrep + cur.execute( + sql.SQL( + """ + DELETE FROM {schema}.{table} + WHERE (index between 10 and 100) + OR (index between 2200 and 2700) + OR (index between 3000 and 3500) + OR (index between 6800 and 7200) + OR (index between 9900 and 10000) + """ + ).format(schema=sql.Identifier(l_schema), table=sql.Identifier(l_table)) + ) + conn.commit() + rows_to_delete = cur.rowcount + cur.close() + conn.close() + + mtree_cli.table_diff("eqn-t9da", table_name=table) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file = match.group(1) + + with open(diff_file, "r") as f: + diff_data = json.load(f) + + assert len(diff_data["diffs"]["n1/n2"]["n2"]) == rows_to_delete + + cli.table_repair( + cluster_name="eqn-t9da", + table_name=table, + diff_file=diff_file, + source_of_truth="n2", + ) + + @pytest.mark.parametrize("table", ["public.customers"]) + def test_non_contiguous_update(self, cli, mtree_cli, capsys, table): + """Test non-contiguous updates""" + l_schema, l_table = table.split(".") + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + + # nosemgrep + cur.execute( + sql.SQL( + """ + UPDATE {schema}.{table} + SET first_name = 'NonContiguousUpdate' + WHERE (index between 20 and 150) + OR (index between 3500 and 4800) + OR (index between 9500 and 10000) + """ + ).format(schema=sql.Identifier(l_schema), table=sql.Identifier(l_table)) + ) + + conn.commit() + rows_to_update = cur.rowcount + cur.close() + conn.close() + + mtree_cli.table_diff("eqn-t9da", table_name=table) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file = match.group(1) + + with open(diff_file, "r") as f: + diff_data = json.load(f) + + assert len(diff_data["diffs"]["n1/n2"]["n1"]) == rows_to_update + for diff in diff_data["diffs"]["n1/n2"]["n1"]: + assert diff["first_name"] == "NonContiguousUpdate" + + cli.table_repair( + cluster_name="eqn-t9da", + table_name=table, + diff_file=diff_file, + source_of_truth="n2", + ) + + def test_various_datatype_pkey(self, mtree_cli, capsys, nodes): + """Test that the trigger function works with various datatypes""" + table_name = "public.datatype_test" + cluster_name = "eqn-t9da" + + fake = Faker() + insert_data = [] + for _ in range(100): + insert_data.append((fake.date_time_this_year(), fake.email(), fake.word())) + + try: + for node in nodes: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + sql.SQL( + """ + CREATE TABLE IF NOT EXISTS public.datatype_test ( + sub_time TIMESTAMP, + email TEXT, + data TEXT, + PRIMARY KEY (sub_time, email) + ); + """ + ) + ) + + cur.executemany( + sql.SQL( + """ + INSERT INTO public.datatype_test (sub_time, email, data) + VALUES (%s, %s, %s) + ON CONFLICT (sub_time, email) DO NOTHING + """ + ), + insert_data, + ) + + cur.execute( + sql.SQL( + """ + SELECT + spock.repset_add_table('test_repset', 'public.datatype_test') + """ + ) + ) + + conn.commit() + cur.close() + conn.close() + + mtree_cli.build( + cluster_name, + table_name=table_name, + block_size=20, + skip_block_size_check=True, + ) + + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + + # nosemgrep + cur.execute( + sql.SQL( + """ + SELECT sub_time, email + FROM public.datatype_test + ORDER BY random() + LIMIT 5 + """ + ) + ) + rows_to_update = cur.fetchall() + + for sub_time, email in rows_to_update: + # nosemgrep + cur.execute( + sql.SQL( + "UPDATE public.datatype_test " + "SET data = 'DataTypeTest' " + "WHERE sub_time = %s AND email = %s" + ), + (sub_time, email), + ) + conn.commit() + cur.close() + conn.close() + + mtree_cli.table_diff(cluster_name, table_name=table_name) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file = match.group(1) + + with open(diff_file, "r") as f: + diff_data = json.load(f) + + assert len(diff_data["diffs"]["n1/n2"]["n1"]) == len(rows_to_update) + for diff in diff_data["diffs"]["n1/n2"]["n1"]: + assert diff["data"] == "DataTypeTest" + + finally: + pass + for node in nodes: + try: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + sql.SQL("DROP TABLE IF EXISTS public.datatype_test CASCADE") + ) + conn.commit() + cur.close() + conn.close() + except Exception as e: + print(f"Cleanup failed for {table_name} on {node}: {e}") + + def test_uuid_pkey_support(self, mtree_cli, capsys, nodes): + """Test that the trigger function works with UUID primary keys""" + table_name = "public.uuid_composite_test" + cluster_name = "eqn-t9da" + + fake = Faker() + insert_data = [] + for _ in range(100): + insert_data.append((fake.uuid4(), fake.email(), fake.word())) + + try: + for node in nodes: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + sql.SQL( + """ + CREATE TABLE IF NOT EXISTS public.uuid_composite_test ( + id UUID, + email TEXT, + data TEXT, + PRIMARY KEY (id, email) + ); + """ + ) + ) + cur.execute( + sql.SQL( + """ + SELECT + spock.repset_add_table( + 'test_repset', + 'public.uuid_composite_test' + ) + """ + ) + ) + + cur.executemany( + sql.SQL( + """ + INSERT INTO public.uuid_composite_test (id, email, data) + VALUES (%s, %s, %s) + ON CONFLICT (id, email) DO NOTHING + """ + ), + insert_data, + ) + conn.commit() + cur.close() + conn.close() + + mtree_cli.build( + cluster_name, + table_name=table_name, + block_size=5, + skip_block_size_check=True, + ) + + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + new_max_uuid = "f" * 32 + new_email = "zzzz@zzzz.com" + # nosemgrep + cur.execute( + sql.SQL( + """ + INSERT INTO public.uuid_composite_test (id, email, data) + VALUES (%s, %s, %s) + """ + ), + (new_max_uuid, new_email, "new max value"), + ) + conn.commit() + cur.close() + conn.close() + + mtree_cli.update( + cluster_name, table_name=table_name, nodes="n1" + ) + captured = capsys.readouterr() + assert "successfully updated" in captured.out.lower() + + mtree_cli.table_diff(cluster_name, table_name=table_name) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + assert "found 1 diffs" in clean_output.lower() + + finally: + for node in nodes: + try: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + sql.SQL( + "DROP TABLE IF EXISTS public.uuid_composite_test CASCADE" + ) + ) + conn.commit() + cur.close() + conn.close() + except Exception as e: + print(f"Cleanup failed for {table_name} on {node}: {e}") + + @pytest.mark.parametrize("table", ["public.customers", "public.customers2"]) + def test_mtree_table_cleanup(self, mtree_cli, capsys, table, nodes): + return super().test_mtree_table_cleanup(mtree_cli, capsys, table, nodes) + + def test_mtree_cleanup(self, mtree_cli, capsys, nodes): + return super().test_mtree_cleanup(mtree_cli, capsys, nodes) diff --git a/cli/scripts/ace-tests/test_merkle_trees_simple.py b/cli/scripts/ace-tests/test_merkle_trees_simple.py new file mode 100644 index 00000000..27ec1114 --- /dev/null +++ b/cli/scripts/ace-tests/test_merkle_trees_simple.py @@ -0,0 +1,701 @@ +import random +import pytest +import re +import json +import abc + +import psycopg +from faker import Faker +from psycopg import sql + +import test_config + + +@pytest.mark.usefixtures("prepare_databases") +@pytest.mark.abstract_base +class TestMerkleTreesSimple(abc.ABC): + """Tests for merkle tree operations""" + + def insert_records(self, conn, schema, table, key_column, num_records): + fake = Faker() + start = 0 + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + inserts = [] + for _ in range(num_records): + start += 1 + index = start + customer_id = fake.uuid4() + first_name = fake.first_name() + last_name = fake.last_name() + company = fake.company() + city = fake.city() + country = fake.country() + phone_1 = fake.phone_number() + phone_2 = fake.phone_number() + email = fake.email() + subscription_date = fake.date_time_this_decade( + before_now=True, after_now=False + ).strftime("%Y-%m-%d %H:%M:%S") + website = fake.url() + + inserts.append( + ( + index, + customer_id, + first_name, + last_name, + company, + city, + country, + phone_1, + phone_2, + email, + subscription_date, + website, + ) + ) + + # nosemgrep + cur.executemany( + sql.SQL( + """ + INSERT INTO {schema}.{table} + ( + index, + customer_id, + first_name, + last_name, + company, + city, + country, + phone_1, + phone_2, + email, + subscription_date, + website + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT({key_column}) DO NOTHING; + """ + ).format( + schema=sql.Identifier(schema), + table=sql.Identifier(table), + key_column=sql.SQL(", ").join(sql.Identifier(k) for k in key_column), + ), + inserts, + ) + + conn.commit() + print(f"Successfully inserted {num_records} records into the database.") + + @pytest.fixture(scope="class", autouse=True) + def setup_test_scenario(self, nodes): + """Setup fixture to create different scenarios for block boundary testing""" + try: + for node in nodes: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + customers2_sql = """ + CREATE TABLE IF NOT EXISTS customers2 ( + index INTEGER PRIMARY KEY NOT NULL, + customer_id TEXT NOT NULL, + first_name TEXT NOT NULL, + last_name TEXT NOT NULL, + company TEXT NOT NULL, + city TEXT NOT NULL, + country TEXT NOT NULL, + phone_1 TEXT NOT NULL, + phone_2 TEXT NOT NULL, + email TEXT NOT NULL, + subscription_date TIMESTAMP WITHOUT TIME ZONE NOT NULL, + website TEXT NOT NULL + ); + """ + + cur.execute(customers2_sql) + + with open(test_config.CUSTOMERS2_CSV, "r") as f: + with cur.copy( + "COPY customers2 FROM STDIN CSV HEADER DELIMITER ','" + ) as copy: + copy.write(f.read()) + + cur.execute( + "SELECT spock.repset_add_table('test_repset', 'customers2')" + ) + + conn.commit() + cur.close() + conn.close() + + print(f"Created customers2 table on {node}") + + yield + + # Cleanup + for node in nodes: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("DROP TABLE customers2 cascade") + conn.commit() + cur.close() + conn.close() + + print(f"Dropped customers2 table on {node}") + + except Exception as e: + pytest.fail(f"Test failed: {str(e)}") + + @pytest.mark.parametrize("cluster_name", ["eqn-t9da"]) + def test_merkle_tree_init(self, mtree_cli, capsys, cluster_name, nodes): + """Test merkle tree init""" + mtree_cli.init(cluster_name) + + captured = capsys.readouterr() + + for node in nodes: + if node == "n1": + node = "localhost" + assert ( + f"Merkle tree objects initialised successfully on {node}" + in captured.out + ) + + @pytest.mark.parametrize("table", ["public.customers", "public.customers2"]) + def test_merkle_tree_setup(self, mtree_cli, capsys, table, nodes): + """ + Test merkle tree setup for tables + """ + + block_size = 50000 if table == "public.customers2" else 1000 + + mtree_cli.build("eqn-t9da", table_name=table, block_size=block_size) + + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + + for node in nodes: + assert f"Merkle tree built successfully on {node}" in clean_output + + @pytest.mark.parametrize( + "table, diff_count", [("public.customers", 75), ("public.customers2", 1000)] + ) + def test_simple_diff(self, cli, mtree_cli, capsys, table, diff_count): + """Test simple diff cases""" + + l_schema, l_table = table.split(".") + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + + update_sql = """ + UPDATE {schema}.{table} + SET first_name = 'Modified' + WHERE index in + (SELECT index from {schema}.{table} order by random() limit %s) + """ + + # nosemgrep + cur.execute( + sql.SQL(update_sql).format( + schema=sql.Identifier(l_schema), + table=sql.Identifier(l_table), + ), + (diff_count,), + ) + + conn.commit() + cur.close() + conn.close() + + mtree_cli.table_diff("eqn-t9da", table_name=table) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file = match.group(1) + + diffs = [] + + with open(diff_file, "r") as f: + diffs = json.load(f) + + assert len(diffs["diffs"]["n1/n2"]["n1"]) == diff_count + assert len(diffs["diffs"]["n1/n2"]["n2"]) == diff_count + + for diff in diffs["diffs"]["n1/n2"]["n1"]: + assert diff["first_name"] == "Modified" + + # We're done. Repair it back. + cli.table_repair( + cluster_name="eqn-t9da", + table_name=table, + diff_file=diff_file, + source_of_truth="n2", + ) + + @pytest.mark.parametrize("table", ["public.customers", "public.customers2"]) + def test_boundaries(self, cli, mtree_cli, capsys, table): + """Test simple diff cases where boundaries are modified""" + + l_schema, l_table = table.split(".") + + conn = psycopg.connect(host="n2", dbname="demo", user="admin") + cur = conn.cursor() + + # nosemgrep + cur.execute( + sql.SQL( + """ + SELECT range_start, range_end + FROM {mtree_table} + where node_level = 0 + order by node_position + """ + ).format( + mtree_table=sql.Identifier(f"ace_mtree_{l_schema}_{l_table}"), + ) + ) + + ranges = cur.fetchall() + + ranges_to_modify = ( + [ranges[0][0]] + + [ + random.choice(r) # nosec: B311 + for r in random.sample(ranges, k=int(0.5 * len(ranges))) # nosec: B311 + ] + + [ranges[-1][0]] + ) + + ranges_to_modify = list(set(ranges_to_modify)) + ranges_to_modify.sort() + + expected_diff_count = len(ranges_to_modify) + + cur.execute("SELECT spock.repair_mode(true)") + for range in ranges_to_modify: + # nosemgrep + cur.execute( + sql.SQL( + """ + UPDATE {table} + SET first_name = 'Modified' + WHERE index = {range} + """ + ).format( + table=sql.Identifier(l_table), + range=sql.SQL(str(range)), + ) + ) + + conn.commit() + cur.close() + conn.close() + + mtree_cli.table_diff("eqn-t9da", table_name=table) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file = match.group(1) + + with open(diff_file, "r") as f: + diff_data = json.load(f) + + assert len(diff_data["diffs"]["n1/n2"]["n2"]) == expected_diff_count + + for diff in diff_data["diffs"]["n1/n2"]["n2"]: + assert diff["first_name"] == "Modified" + + # Repair everything now + cli.table_repair( + cluster_name="eqn-t9da", + table_name=table, + diff_file=diff_file, + source_of_truth="n1", + ) + + @pytest.mark.parametrize("table", ["public.customers"]) + def test_split_ranges(self, cli, mtree_cli, capsys, table): + """ + Testing range splits automatically tests merges as well. + """ + + l_schema, l_table = table.split(".") + + # We'll first delete keys en masse. Then get the new ranges, and finally + # insert records to trigger the split. + conn = psycopg.connect(host="n2", dbname="demo", user="admin") + cur = conn.cursor() + + cur.execute(sql.SQL("SELECT spock.repair_mode(true)")) + # nosemgrep + cur.execute( + sql.SQL("DELETE FROM {schema}.{table} WHERE index <= 4000").format( + schema=sql.Identifier(l_schema), table=sql.Identifier(l_table) + ) + ) + + conn.commit() + cur.close() + conn.close() + + # We're going to deliberately rebalance the tree to make way for the splits. + mtree_cli.update("eqn-t9da", table_name=table, nodes="n2", rebalance=True) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + print(clean_output) + + assert "successfully updated" in clean_output.lower() + + conn = psycopg.connect(host="n2", dbname="demo", user="admin") + cur = conn.cursor() + + # nosemgrep + cur.execute( + sql.SQL( + """ + SELECT range_end + FROM {mtree_table} + where node_level = 0 + order by node_position + limit 1 + """ + ).format( + mtree_table=sql.Identifier(f"ace_mtree_{l_schema}_{l_table}"), + ) + ) + + range_end = cur.fetchone()[0] + assert range_end > 4000 + + # Now we'll insert records to trigger the split. + self.insert_records(conn, l_schema, l_table, ["index"], 2000) + + mtree_cli.update("eqn-t9da", table_name=table, nodes="n2", rebalance=True) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + print(clean_output) + assert "successfully updated" in clean_output.lower() + + # check if the split has happened + # nosemgrep + cur.execute( + sql.SQL( + """ + SELECT range_end + FROM {mtree_table} + where node_level = 0 + order by node_position + limit 1 + """ + ).format( + mtree_table=sql.Identifier(f"ace_mtree_{l_schema}_{l_table}"), + ) + ) + + range_end = cur.fetchone()[0] + assert range_end <= 2000 + + mtree_cli.table_diff("eqn-t9da", table_name=table) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + diff_file = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert diff_file, "Diff file path not found in output" + diff_file = diff_file.group(1) + + cli.table_repair( + cluster_name="eqn-t9da", + table_name=table, + diff_file=diff_file, + source_of_truth="n1", + ) + + @pytest.mark.parametrize("table, block_size", [("public.customers2", 50000)]) + def test_merges(self, cli, mtree_cli, capsys, table, block_size): + """Test merging smaller blocks""" + + l_schema, l_table = table.split(".") + + conn = psycopg.connect(host="n2", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute(sql.SQL("SELECT spock.repair_mode(true)")) + + # delete 100k-200k and >=900k + # nosemgrep + cur.execute( + sql.SQL( + """ + DELETE FROM {schema}.{table} + WHERE (index >= 100000 AND index <= 200000) OR index >= 900000 + """ + ).format(schema=sql.Identifier(l_schema), table=sql.Identifier(l_table)) + ) + + conn.commit() + cur.close() + + mtree_cli.table_diff("eqn-t9da", table_name=table, rebalance=True) + + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + diff_file = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert diff_file, "Diff file path not found in output" + + diff_file = diff_file.group(1) + + conn = psycopg.connect(host="n2", dbname="demo", user="admin") + cur = conn.cursor() + + # nosemgrep + cur.execute( + sql.SQL( + """ + SELECT count(t.index) + FROM {mtree_table} mt + LEFT JOIN {schema}.{table} t + ON t.index >= mt.range_start + AND (t.index <= mt.range_end OR mt.range_end IS NULL) + WHERE mt.node_level = 0 + GROUP BY mt.range_start, mt.range_end + """ + ).format( + schema=sql.Identifier(l_schema), + table=sql.Identifier(l_table), + mtree_table=sql.Identifier(f"ace_mtree_{l_schema}_{l_table}"), + ) + ) + + counts = cur.fetchall() + assert all(count[0] > 0 for count in counts) + + cur.close() + conn.close() + + cli.table_repair( + cluster_name="eqn-t9da", + table_name=table, + diff_file=diff_file, + source_of_truth="n1", + ) + + @pytest.mark.parametrize("table", ["public.customers"]) + def test_non_contiguous_delete(self, cli, mtree_cli, capsys, table): + """Test non-contiguous deletes only mark affected blocks as dirty""" + + l_schema, l_table = table.split(".") + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + + ids_to_delete = [10, 5000, 9900] + # nosemgrep + cur.execute( + sql.SQL( + """ + DELETE FROM {schema}.{table} + WHERE index = ANY(%s) + """ + ).format( + schema=sql.Identifier(l_schema), + table=sql.Identifier(l_table), + ), + (ids_to_delete,), + ) + + conn.commit() + cur.close() + conn.close() + + mtree_cli.table_diff("eqn-t9da", table_name=table) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file = match.group(1) + + with open(diff_file, "r") as f: + diff_data = json.load(f) + + assert len(diff_data["diffs"]["n1/n2"]["n2"]) == len(ids_to_delete) + + cli.table_repair( + cluster_name="eqn-t9da", + table_name=table, + diff_file=diff_file, + source_of_truth="n2", + ) + + @pytest.mark.parametrize("table", ["public.customers"]) + def test_non_contiguous_update(self, cli, mtree_cli, capsys, table): + """Test non-contiguous updates""" + l_schema, l_table = table.split(".") + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + + ids_to_update = [20, 6000, 9800] + # nosemgrep + cur.execute( + sql.SQL( + """ + UPDATE {schema}.{table} + SET first_name = 'NonContiguous' + WHERE index = ANY(%s) + """ + ).format( + schema=sql.Identifier(l_schema), + table=sql.Identifier(l_table), + ), + (ids_to_update,), + ) + + conn.commit() + cur.close() + conn.close() + + mtree_cli.table_diff("eqn-t9da", table_name=table) + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file = match.group(1) + + with open(diff_file, "r") as f: + diff_data = json.load(f) + + assert len(diff_data["diffs"]["n1/n2"]["n1"]) == len(ids_to_update) + assert len(diff_data["diffs"]["n1/n2"]["n2"]) == len(ids_to_update) + + for diff in diff_data["diffs"]["n1/n2"]["n1"]: + assert diff["first_name"] == "NonContiguous" + + cli.table_repair( + cluster_name="eqn-t9da", + table_name=table, + diff_file=diff_file, + source_of_truth="n2", + ) + + def test_uuid_pkey_support(self, cli, mtree_cli, capsys, nodes): + """Test that the trigger function works with UUID primary keys""" + table_name = "public.uuid_test" + cluster_name = "eqn-t9da" + + try: + for node in nodes: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + """ + CREATE TABLE IF NOT EXISTS public.uuid_test ( + id UUID PRIMARY KEY, + data TEXT + ); + """ + ) + cur.execute( + sql.SQL( + "SELECT spock.repset_add_table('test_repset', " + "'public.uuid_test')" + ) + ) + conn.commit() + + if node == "n1": + fake = Faker() + for _ in range(10): + cur.execute( + sql.SQL( + """ + INSERT INTO public.uuid_test (id, data) + VALUES (%s, %s) + """ + ), + (fake.uuid4(), fake.word()), + ) + conn.commit() + + cur.close() + conn.close() + + mtree_cli.build( + cluster_name, + table_name=table_name, + block_size=5, + skip_block_size_check=True, + ) + + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + fake = Faker() + new_max_uuid = "f" * 32 + # nosemgrep + cur.execute( + sql.SQL("INSERT INTO public.uuid_test (id, data) VALUES (%s, %s)"), + (new_max_uuid, "new max value"), + ) + conn.commit() + cur.close() + conn.close() + + mtree_cli.update(cluster_name, table_name=table_name, nodes="n1") + captured = capsys.readouterr() + assert "successfully updated" in captured.out.lower() + + finally: + # 5. Cleanup + for node in nodes: + try: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + cur.execute(sql.SQL("DROP TABLE public.uuid_test CASCADE")) + conn.commit() + cur.close() + conn.close() + except Exception as e: + print(f"Cleanup failed for {table_name} on {node}: {e}") + + @pytest.mark.parametrize("table", ["public.customers", "public.customers2"]) + def test_mtree_table_cleanup(self, mtree_cli, capsys, table, nodes): + """Test merkle tree table cleanup""" + mtree_cli.teardown("eqn-t9da", table_name=table) + captured = capsys.readouterr() + + for node in nodes: + if node == "n1": + node = "localhost" + assert ( + f"Dropped table-specific merkle tree objects on {node}" in captured.out + ) + + def test_mtree_cleanup(self, mtree_cli, capsys, nodes): + mtree_cli.teardown("eqn-t9da") + + captured = capsys.readouterr() + + for node in nodes: + if node == "n1": + node = "localhost" + assert f"Dropped generic merkle tree objects on {node}" in captured.out diff --git a/cli/scripts/ace-tests/test_mixed_case.py b/cli/scripts/ace-tests/test_mixed_case.py index 7914d2f1..c56d1f29 100644 --- a/cli/scripts/ace-tests/test_mixed_case.py +++ b/cli/scripts/ace-tests/test_mixed_case.py @@ -100,9 +100,7 @@ def test_table_diff_with_differences( @pytest.mark.parametrize("table_name", ["public.CuStOmErS"]) def test_simple_table_repair(self, cli, capsys, table_name, diff_file_path): - return super().test_simple_table_repair( - cli, capsys, table_name, diff_file_path - ) + return super().test_simple_table_repair(cli, capsys, table_name, diff_file_path) @pytest.mark.parametrize("table_name", ["public.CuStOmErS"]) @pytest.mark.parametrize("key_column", ["index"]) @@ -112,11 +110,3 @@ def test_table_rerun_temptable( return super().test_table_rerun_temptable( cli, capsys, ace_conf, table_name, key_column, diff_file_path ) - - @pytest.mark.parametrize("table_name", ["public.CuStOmErS"]) - def test_table_rerun_multiprocessing( - self, cli, capsys, table_name, diff_file_path - ): - return super().test_table_rerun_multiprocessing( - cli, capsys, table_name, diff_file_path - ) diff --git a/cli/scripts/ace-tests/test_replication.py b/cli/scripts/ace-tests/test_replication.py index 2a8972bd..bfeb2dbc 100644 --- a/cli/scripts/ace-tests/test_replication.py +++ b/cli/scripts/ace-tests/test_replication.py @@ -74,7 +74,10 @@ def test_replication( time.sleep(2) # Execute table diff and verify results - cli.table_diff_cli("eqn-t9da", table_name) + cli.table_diff( + cluster_name="eqn-t9da", + table_name=table_name, + ) # Capture and verify output captured = capsys.readouterr() diff --git a/cli/scripts/ace-tests/test_repset_diff.py b/cli/scripts/ace-tests/test_repset_diff.py index 92ea8daf..04bb4a77 100644 --- a/cli/scripts/ace-tests/test_repset_diff.py +++ b/cli/scripts/ace-tests/test_repset_diff.py @@ -2,14 +2,57 @@ import re import pytest import psycopg -from test_simple import TestSimple -@pytest.mark.skip(reason="Will revisit this in a bit") @pytest.mark.usefixtures("prepare_databases") -class TestRepsetDiff(TestSimple): +class TestRepsetDiff(): """Test class for repset-diff functionality""" + @pytest.fixture(scope="class") + def setup_test_scenario(self, nodes): + """Setup fixture to create different scenarios for repset-diff testing""" + try: + for node in nodes: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + + cur.execute( + """ + CREATE TABLE repset_diff_test + ( + id SERIAL PRIMARY KEY, + name VARCHAR(255) + ) + """ + ) + cur.execute( + """ + INSERT INTO repset_diff_test (name) + VALUES ('Alice'), ('Bob'), ('Charlie') + """ + ) + conn.commit() + cur.close() + conn.close() + + print(f"Created repset_diff_test table on {node}") + + yield + + for node in nodes: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + + cur.execute("DROP TABLE repset_diff_test") + conn.commit() + cur.close() + conn.close() + + print(f"Dropped repset_diff_test table on {node}") + + except Exception as e: + pytest.fail(f"Test failed: {str(e)}") + def _introduce_differences( self, ace_conf, node, table_name, column_name, key_column ): @@ -30,6 +73,8 @@ def _introduce_differences( conn = psycopg.connect(**params) cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + # Modify 50 rows in the table cur.execute( f""" @@ -61,7 +106,10 @@ def test_basic_repset_diff(self, cli, capsys, ace_conf, diff_file_path): ) # Run repset-diff - cli.repset_diff_cli("eqn-t9da", "demo_replication_set") + cli.repset_diff( + cluster_name="eqn-t9da", + repset_name="test_repset", + ) # Capture and clean the output captured = capsys.readouterr() @@ -71,7 +119,7 @@ def test_basic_repset_diff(self, cli, capsys, ace_conf, diff_file_path): # Verify differences were found assert ( - "differences found" in clean_output.lower() + "tables do not match" in clean_output.lower() ), "No differences detected" # Get the diff file path @@ -84,9 +132,10 @@ def test_basic_repset_diff(self, cli, capsys, ace_conf, diff_file_path): diff_data = json.load(f) # Verify the number of differences matches our modifications - assert ( - len(diff_data["diffs"]["n1/n2"]["n2"]) == 50 - ), "Expected 50 differences" + assert len(diff_data["diffs"]["n1/n2"]["n2"]) == len( + modified_indices + ), f"Expected {len(modified_indices)} differences," + f" found {len(diff_data['diffs']['n1/n2']['n2'])}" # Verify each modified row is present in the diff diff_indices = {diff["index"] for diff in diff_data["diffs"]["n1/n2"]["n2"]} @@ -106,8 +155,10 @@ def test_repset_diff_skip_tables(self, cli, capsys, ace_conf): ) # Run repset-diff with customers table skipped - cli.repset_diff_cli( - "eqn-t9da", "demo_replication_set", skip_tables=["public.customers"] + cli.repset_diff( + cluster_name="eqn-t9da", + repset_name="test_repset", + skip_tables="public.customers", ) # Capture and clean the output @@ -120,14 +171,13 @@ def test_repset_diff_skip_tables(self, cli, capsys, ace_conf): assert ( "skipping table public.customers" in clean_output.lower() ), "Table not skipped" - assert ( - "tables match ok" in clean_output.lower() - ), "Differences found in non-skipped tables" except Exception as e: pytest.fail(f"Test failed: {str(e)}") - def test_repset_diff_skip_file(self, cli, capsys, ace_conf, tmp_path): + def test_repset_diff_skip_file( + self, cli, capsys, ace_conf, tmp_path, diff_file_path + ): """Test repset-diff with skip-file option""" try: # Create a skip file @@ -140,8 +190,10 @@ def test_repset_diff_skip_file(self, cli, capsys, ace_conf, tmp_path): ) # Run repset-diff with skip file - cli.repset_diff_cli( - "eqn-t9da", "demo_replication_set", skip_file=str(skip_file) + cli.repset_diff( + cluster_name="eqn-t9da", + repset_name="test_repset", + skip_file=str(skip_file), ) # Capture and clean the output @@ -154,9 +206,14 @@ def test_repset_diff_skip_file(self, cli, capsys, ace_conf, tmp_path): assert ( "skipping table public.customers" in clean_output.lower() ), "Table not skipped" - assert ( - "tables match ok" in clean_output.lower() - ), "Differences found in non-skipped tables" + + # Cleanup by repairing + cli.table_repair( + cluster_name="eqn-t9da", + table_name="public.customers", + diff_file=diff_file_path.path, + source_of_truth="n1", + ) except Exception as e: pytest.fail(f"Test failed: {str(e)}") diff --git a/cli/scripts/ace-tests/test_simple.py b/cli/scripts/ace-tests/test_simple.py index eca662b9..ecd2c47e 100644 --- a/cli/scripts/ace-tests/test_simple.py +++ b/cli/scripts/ace-tests/test_simple.py @@ -41,7 +41,10 @@ def test_database_connectivity(self, ace_conf, nodes): @pytest.mark.parametrize("table_name", ["public.customers"]) def test_simple_table_diff(self, cli, capsys, table_name): """Test table diff on cluster eqn-t9da for specified table""" - cli.table_diff_cli("eqn-t9da", table_name) + cli.table_diff( + cluster_name="eqn-t9da", + table_name=table_name, + ) captured = capsys.readouterr() output = captured.out assert ( @@ -69,7 +72,10 @@ def test_table_diff_with_differences( ) # Execute table diff and verify results - cli.table_diff_cli("eqn-t9da", table_name) + cli.table_diff( + cluster_name="eqn-t9da", + table_name=table_name, + ) # Capture the output captured = capsys.readouterr() @@ -119,10 +125,10 @@ def test_table_diff_with_differences( @pytest.mark.parametrize("table_name", ["public.customers"]) def test_simple_table_repair(self, cli, capsys, table_name, diff_file_path): """Test table repair on cluster eqn-t9da for specified table""" - cli.table_repair_cli( - "eqn-t9da", - table_name, - diff_file_path.path, + cli.table_repair( + cluster_name="eqn-t9da", + table_name=table_name, + diff_file=diff_file_path.path, source_of_truth="n1", ) @@ -133,7 +139,10 @@ def test_simple_table_repair(self, cli, capsys, table_name, diff_file_path): ), f"Table repair failed. Output: {output}" # Verify the table is repaired - cli.table_diff_cli("eqn-t9da", table_name) + cli.table_diff( + cluster_name="eqn-t9da", + table_name=table_name, + ) captured = capsys.readouterr() output = captured.out @@ -186,7 +195,10 @@ def test_table_rerun_temptable( pytest.fail(f"Failed to introduce diffs on n2: {str(e)}") # Running the table-diff to get the diff file - cli.table_diff_cli("eqn-t9da", table_name) + cli.table_diff( + cluster_name="eqn-t9da", + table_name=table_name, + ) # Capture the output captured = capsys.readouterr() @@ -199,13 +211,11 @@ def test_table_rerun_temptable( assert match, "Diff file path not found in output" path = match.group(1) - cli.table_rerun_cli( - "eqn-t9da", - path, - table_name, - "demo", # dbname - False, # quiet - "hostdb", # behaviour + cli.table_rerun( + cluster_name="eqn-t9da", + diff_file=path, + table_name=table_name, + dbname="demo", ) rerun_captured = capsys.readouterr() @@ -232,49 +242,12 @@ def test_table_rerun_temptable( for diff in diff_data["diffs"]["n1/n2"]["n2"]: assert diff["city"] == "Casablanca", "Expected city to be 'Casablanca'" - @pytest.mark.parametrize("table_name", ["public.customers"]) - def test_table_rerun_multiprocessing(self, cli, capsys, table_name, diff_file_path): - """Test table rerun multiprocessing on cluster eqn-t9da""" - - # We have already introduced diffs in the previous test. - # We will now rerun the diffs using multiprocessing - - cli.table_rerun_cli( - "eqn-t9da", - diff_file_path.path, - table_name, - "demo", # dbname - False, # quiet - "multiprocessing", # behaviour - ) - - captured = capsys.readouterr() - output = captured.out - - clean_output = re.sub(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", output) - - match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) - - assert match, "Diff file path not found in output" - diff_file_path.path = match.group(1) - - # Read the diff file into a dictionary - with open(diff_file_path.path, "r") as f: - diff_data = json.load(f) - - # Verify we still have 100 diffs - assert len(diff_data["diffs"]["n1/n2"]["n2"]) == 100, "Expected 100 differences" - - # Verify the diffs are correct - for diff in diff_data["diffs"]["n1/n2"]["n2"]: - assert diff["city"] == "Casablanca", "Expected city to be 'Casablanca'" - # Now that we have verified diffs through both methods, we can use repair # to restore the state of the table - cli.table_repair_cli( - "eqn-t9da", - table_name, - diff_file_path.path, + cli.table_repair( + cluster_name="eqn-t9da", + table_name=table_name, + diff_file=diff_file_path.path, source_of_truth="n1", ) diff --git a/cli/scripts/ace-tests/test_simple_base.py b/cli/scripts/ace-tests/test_simple_base.py index 3e3a400f..d239277e 100644 --- a/cli/scripts/ace-tests/test_simple_base.py +++ b/cli/scripts/ace-tests/test_simple_base.py @@ -123,10 +123,3 @@ def test_table_rerun_temptable( ): """Test table rerun temptable on cluster eqn-t9da for specified table""" pass - - @abc.abstractmethod - @pytest.mark.skip(reason="Abstract base class method") - @pytest.mark.parametrize("table_name", ["public.customers"]) - def test_table_rerun_multiprocessing(self, cli, capsys, table_name, diff_file_path): - """Test table rerun multiprocessing on cluster eqn-t9da""" - pass diff --git a/cli/scripts/ace-tests/test_table_filter.py b/cli/scripts/ace-tests/test_table_filter.py index 938ff682..35346505 100644 --- a/cli/scripts/ace-tests/test_table_filter.py +++ b/cli/scripts/ace-tests/test_table_filter.py @@ -15,7 +15,11 @@ def test_simple_table_diff_with_filter(self, cli, capsys, table_name, table_filt """Test table diff with filter when no differences exist""" expected_total_rows_checked = 297 - cli.table_diff_cli("eqn-t9da", table_name, table_filter=table_filter) + cli.table_diff( + cluster_name="eqn-t9da", + table_name=table_name, + table_filter=table_filter, + ) captured = capsys.readouterr() output = captured.out @@ -69,7 +73,11 @@ def test_table_diff_with_filter_and_differences( expected_total_rows_checked = 297 # Run table-diff with filter - cli.table_diff_cli("eqn-t9da", table_name, table_filter=table_filter) + cli.table_diff( + cluster_name="eqn-t9da", + table_name=table_name, + table_filter=table_filter, + ) captured = capsys.readouterr() output = captured.out @@ -116,7 +124,11 @@ def test_table_diff_with_filter_excluding_differences( try: # Differences from previous test should still exist in index < 100 # Run table-diff with filter that excludes those rows - cli.table_diff_cli("eqn-t9da", table_name, table_filter=table_filter) + cli.table_diff( + cluster_name="eqn-t9da", + table_name=table_name, + table_filter=table_filter, + ) captured = capsys.readouterr() output = captured.out expected_total_rows_checked = 29703 # 9901 x 3 @@ -146,10 +158,10 @@ def test_table_repair_with_filter( ): """Test table repair with filter""" # Run repair with the same filter - cli.table_repair_cli( - "eqn-t9da", - table_name, - diff_file_path.path, + cli.table_repair( + cluster_name="eqn-t9da", + table_name=table_name, + diff_file=diff_file_path.path, source_of_truth="n1", ) @@ -163,7 +175,11 @@ def test_table_repair_with_filter( ), f"Table repair failed. Output: {output}" # Verify repair worked within filter range - cli.table_diff_cli("eqn-t9da", table_name, table_filter=table_filter) + cli.table_diff( + cluster_name="eqn-t9da", + table_name=table_name, + table_filter=table_filter, + ) captured = capsys.readouterr() output = captured.out assert ( @@ -195,7 +211,11 @@ def test_table_rerun_with_filter( pytest.fail(f"Failed to introduce diffs for rerun test: {str(e)}") # Get diff file with filter - cli.table_diff_cli("eqn-t9da", table_name, table_filter=table_filter) + cli.table_diff( + cluster_name="eqn-t9da", + table_name=table_name, + table_filter=table_filter, + ) captured = capsys.readouterr() clean_output = re.sub( r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out @@ -204,42 +224,30 @@ def test_table_rerun_with_filter( assert match, "Diff file path not found in output" path = match.group(1) - # Test both rerun behaviors with filter - for behavior in ["hostdb", "multiprocessing"]: - cli.table_rerun_cli( - "eqn-t9da", - path, - table_name, - "demo", - False, - behavior, - ) - captured = capsys.readouterr() - output = captured.out - clean_output = re.sub(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", output) + cli.table_rerun( + cluster_name="eqn-t9da", + diff_file=path, + table_name=table_name, + dbname="demo", + ) + captured = capsys.readouterr() + output = captured.out + clean_output = re.sub(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", output) - match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) - assert match, "Diff file path not found in output" - diff_file_path.path = match.group(1) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file_path.path = match.group(1) - assert ( - "written out to" in clean_output.lower() - ), f"Table rerun failed for {behavior}" + assert "written out to" in clean_output.lower(), "Table rerun failed" - # Verify diffs are correct - with open(diff_file_path.path, "r") as f: - diff_data = json.load(f) + # Verify diffs are correct + with open(diff_file_path.path, "r") as f: + diff_data = json.load(f) - assert ( - len(diff_data["diffs"]["n1/n2"]["n2"]) == 49 - ), f"Expected 49 differences for {behavior}" - for diff in diff_data["diffs"]["n1/n2"]["n2"]: - assert ( - diff["index"] < 100 - ), f"Found diff outside filter range in {behavior}" - assert ( - diff["city"] == "FilterCity" - ), f"Unexpected city value in {behavior}" + assert len(diff_data["diffs"]["n1/n2"]["n2"]) == 49, "Expected 49 differences" + for diff in diff_data["diffs"]["n1/n2"]["n2"]: + assert diff["index"] < 100, "Found diff outside filter range" + assert diff["city"] == "FilterCity", "Unexpected city value" @pytest.mark.parametrize("table_name", ["public.customers"]) @pytest.mark.parametrize("table_filter", ["index < 100"]) @@ -252,7 +260,11 @@ def test_table_diff_with_filter_cleanup_on_worker_failure( mock_compare.side_effect = Exception("Simulated worker failure") with pytest.raises(SystemExit): - cli.table_diff_cli("eqn-t9da", table_name, table_filter=table_filter) + cli.table_diff( + cluster_name="eqn-t9da", + table_name=table_name, + table_filter=table_filter, + ) for host in ["n1", "n2", "n3"]: try: diff --git a/cli/scripts/ace.py b/cli/scripts/ace.py index b454affe..ecf50d43 100755 --- a/cli/scripts/ace.py +++ b/cli/scripts/ace.py @@ -12,7 +12,9 @@ from datetime import datetime import logging from itertools import chain +from psycopg import ClientCursor +from ace_sql import ESTIMATE_ROW_COUNT, GET_PKEY_OFFSETS import fire import psycopg from psycopg.rows import dict_row @@ -23,6 +25,7 @@ import ace_db import ace_config as config from ace_data_models import ( + MerkleTreeTask, RepsetDiffTask, SchemaDiffTask, SpockDiffTask, @@ -66,16 +69,29 @@ def write_pg_dump(p_ip, p_db, p_port, p_prfx, p_schm, p_base_dir="/tmp"): return out_file -""" -Accepts a connection object and returns the version of spock installed +def print_query(conn, query, params=None): + client_cur = ClientCursor(conn) + print(client_cur.mogrify(query, params)) -@param: conn - connection object -@return: float - version of spock installed -""" +def sanitise_input(input: str) -> str: + """ + Sanitises input to ensure it is a valid identifier. + """ + if not re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", input): + raise ValueError(f"Invalid identifier: {input}") + + return input def get_spock_version(conn): + """ + Accepts a connection object and returns the version of spock installed + + @param: conn - connection object + @return: float - version of spock installed + + """ data = [] sql = "SELECT spock.spock_version();" try: @@ -123,16 +139,23 @@ def fix_schema(diff_file, sql1, sql2): return 1 -def get_row_count(p_con, p_schema, p_table): - sql = f'SELECT count(*) FROM {p_schema}."{p_table}"' +def get_row_count_estimate(p_con, p_schema, p_table): + """ + Returns an estimate of the number of rows in a table. + Note: This cannot be used for non-materialised views. + """ try: cur = p_con.cursor() - cur.execute(sql) + cur.execute( + sql.SQL(ESTIMATE_ROW_COUNT).format( + schema=sql.Literal(p_schema), table=sql.Literal(p_table) + ) + ) r = cur.fetchone() cur.close() except Exception as e: - util.exit_message("Error in get_row_count():\n" + str(e), 1) + util.exit_message("Error in get_row_count_estimate():\n" + str(e), 1) if not r: return 0 @@ -142,6 +165,29 @@ def get_row_count(p_con, p_schema, p_table): return rows +def get_row_count(p_con, p_schema, p_table): + """ + Returns the actual number of rows in a table. + """ + + try: + cur = p_con.cursor() + cur.execute( + f""" + SELECT COUNT(*) FROM "{p_schema}"."{p_table}" + """ + ) + r = cur.fetchone() + cur.close() + except Exception as e: + util.exit_message("Error in get_row_count():\n" + str(e), 1) + + if not r: + return 0 + + return int(r[0]) + + def get_cols(p_con, p_schema, p_table): sql = """ SELECT @@ -343,6 +389,129 @@ def check_cluster_exists(cluster_name, base_dir="cluster"): return True if os.path.exists(cluster_dir) else False +def ensure_pgcrypto_installed(conn: psycopg.Connection): + """ + Ensure that the pgcrypto extension is installed on the database. + ACE needs the digest() function for computing row checksums. + """ + + with conn.cursor() as cur: + cur.execute("CREATE EXTENSION IF NOT EXISTS pgcrypto;") + conn.commit() + + +def generate_pkey_offsets_query( + schema, table, key_columns, table_sample_method, sample_percent, ntile_count +): + key_columns_select = ",\n ".join(key_columns) + key_columns_order = ", ".join(key_columns) + key_columns_order_desc = ", ".join(f"{col} DESC" for col in key_columns) + + first_row_selects = ",\n ".join( + f"(SELECT {col} FROM first_row) as {col}" for col in key_columns + ) + + last_row_selects = ",\n ".join( + f"(SELECT {col} FROM last_row) as {col}" for col in key_columns + ) + + first_row_tuple_selects = ",\n ".join( + f"(SELECT {col} FROM first_row)" for col in key_columns + ) + + range_start_columns = ",\n ".join( + f"{col} as range_start_{col}" for col in key_columns + ) + + range_end_columns = ",\n ".join( + f"LEAD({col}) OVER (ORDER BY seq, {key_columns_order}) as range_end_{col}" + for col in key_columns + ) + + range_output_columns = ",\n ".join( + f"range_start_{col},\n range_end_{col}" for col in key_columns + ) + + full_query = sql.SQL(GET_PKEY_OFFSETS).format( + key_columns_select=sql.SQL(key_columns_select), + schema=sql.Identifier(schema), + table=sql.Identifier(table), + table_sample_method=sql.SQL(table_sample_method), + sample_percent=sql.SQL(str(sample_percent)), + ntile_count=sql.Literal(ntile_count), + key_columns_order=sql.SQL(key_columns_order), + key_columns_order_desc=sql.SQL(key_columns_order_desc), + first_row_selects=sql.SQL(first_row_selects), + last_row_selects=sql.SQL(last_row_selects), + first_row_tuple_selects=sql.SQL(first_row_tuple_selects), + range_start_columns=sql.SQL(range_start_columns), + range_end_columns=sql.SQL(range_end_columns), + range_output_columns=sql.SQL(range_output_columns), + ) + + return full_query + + +def process_pkey_offsets(offsets: list): + + pkey_offsets = [] + simple_primary_key = len(offsets[0]) == 2 + + if simple_primary_key: + pkey_offsets.append(([None], [offsets[0][0]])) + else: + starts = tuple(None for _ in range(len(offsets[0]) // 2)) + ends = tuple(offsets[0][i] for i, _ in enumerate(offsets[0]) if i % 2 == 0) + pkey_offsets.append(([starts], [ends])) + + for offset in offsets: + if simple_primary_key: + pkey_offsets.append(([offset[0]], [offset[1]])) + else: + starts = tuple(r for i, r in enumerate(offset) if i % 2 == 0) + ends = tuple(r for i, r in enumerate(offset) if i % 2 == 1) + pkey_offsets.append(([starts], [ends])) + + return pkey_offsets + + +def compute_sampling_parameters(row_count): + """ + Calculate sampling parameters (sample_method and sample_percent) based on total + rows. + + Previously, we used to get all primary keys in a sorted order and then compute + the block ranges (or offsets) for the table. However, this method turned out + to be very expensive. Therefore, we now use the tablesample method to sample the + primary keys and then compute the block ranges. + + However, even with sampling, we need to carefully choose the sample method and + percent to ensure that the block ranges are computed correctly, i.e., the + sampled offsets are somewhat uniformly distributed across the keyspace of the + table. The bernoulli method is a good choice for tables with a small number of + rows, while the system method is good for other cases. + """ + + sample_method = "BERNOULLI" + sample_percent = 100 + + if row_count <= 10**4: + return sample_method, sample_percent + + if row_count <= 10**5: + sample_percent = 10 + elif row_count <= 10**6: + sample_percent = 1 + elif row_count <= 10**8: + sample_method = "SYSTEM" + sample_percent = 0.1 + else: + sample_method = "SYSTEM" + sample_percent = 0.01 + + return sample_method, sample_percent + + def check_user_privileges(conn, username, schema, table, required_privileges=[]): """ In most cases, the ace user provisioned in production will have limited access @@ -768,7 +937,7 @@ def write_diffs_json(td_task, diff_dict, col_types, quiet_mode=False): "schema_name": td_task._table_name.split(".")[0], "table_name": td_task._table_name.split(".")[1], "nodes": td_task._nodes, - "block_rows": td_task.block_rows, + "block_size": td_task.block_size, "max_cpu_ratio": td_task.max_cpu_ratio, "batch_size": td_task.batch_size, "start_time": td_task.scheduler.started_at, @@ -849,26 +1018,30 @@ def validate_table_diff_inputs(td_task: TableDiffTask) -> None: Validates the basic inputs for a table diff task without establishing connections. Raises AceException if validation fails. """ - if not td_task.cluster_name or not td_task._table_name: + if not td_task.cluster_name or td_task.cluster_name == "": raise AceException("cluster_name and table_name are required arguments") - if type(td_task.block_rows) is str: + if not td_task._table_name or td_task._table_name == "": + raise AceException("table_name is a required argument") + + if type(td_task.block_size) is str: try: - td_task.block_rows = int(td_task.block_rows) + td_task.block_size = int(td_task.block_size) except Exception: - raise AceException("Invalid values for ACE_BLOCK_ROWS") - elif type(td_task.block_rows) is not int: - raise AceException("Invalid value type for ACE_BLOCK_ROWS") + raise AceException("Invalid values for DIFF_BLOCK_SIZE") + elif type(td_task.block_size) is not int: + raise AceException("Invalid value type for DIFF_BLOCK_SIZE") - # Capping max block size here to prevent the hash function from taking forever - if td_task.block_rows > config.MAX_ALLOWED_BLOCK_SIZE: - raise AceException( - f"Block row size should be <= {config.MAX_ALLOWED_BLOCK_SIZE}" - ) - if td_task.block_rows < config.MIN_ALLOWED_BLOCK_SIZE: - raise AceException( - f"Block row size should be >= {config.MIN_ALLOWED_BLOCK_SIZE}" - ) + if not td_task._override_block_size: + # Capping max block size here to prevent the hash function from taking forever + if td_task.block_size > config.MAX_DIFF_BLOCK_SIZE: + raise AceException( + f"Block row size should be <= {config.MAX_DIFF_BLOCK_SIZE}" + ) + if td_task.block_size < config.MIN_DIFF_BLOCK_SIZE: + raise AceException( + f"Block row size should be >= {config.MIN_DIFF_BLOCK_SIZE}" + ) if type(td_task.max_cpu_ratio) is int: td_task.max_cpu_ratio = float(td_task.max_cpu_ratio) @@ -922,8 +1095,10 @@ def validate_table_diff_inputs(td_task: TableDiffTask) -> None: raise AceException( f"TableName {td_task._table_name} must be of form" " 'schema.table_name'" ) - l_schema = nm_lst[0] - l_table = nm_lst[1] + l_schema, l_table = nm_lst + + l_schema = sanitise_input(l_schema) + l_table = sanitise_input(l_table) db, pg, node_info = cluster.load_json(td_task.cluster_name) @@ -1003,14 +1178,16 @@ def table_diff_checks( user = node_info["db_user"] port = node_info.get("port", 5432) - params, conn = td_task.connection_pool.get_cluster_node_connection( - node_info, - td_task.cluster_name, - invoke_method=td_task.invoke_method, - client_role=( - td_task.client_role if td_task.invoke_method == "api" else None - ), - ) + if node_info["name"] in td_task.fields.node_list: + params, conn = td_task.connection_pool.get_cluster_node_connection( + node_info, + client_role=( + td_task.client_role if td_task.invoke_method == "api" else None + ), + ) + ensure_pgcrypto_installed(conn) + else: + continue curr_cols = get_cols(conn, l_schema, l_table) curr_key = get_key(conn, l_schema, l_table) @@ -1100,15 +1277,21 @@ def table_diff_checks( conn.commit() # Now, we need to check if the view actually has any rows - view_sql = sql.SQL("SELECT COUNT(*) FROM {view_name}").format( + view_sql = sql.SQL( + """ + SELECT EXISTS ( + SELECT 1 FROM {view_name} + ) AS has_rows + """ + ).format( view_name=sql.Identifier( f"{td_task.scheduler.task_id}_{l_table}_filtered" ), ) cur = conn.cursor() cur.execute(view_sql) - row_count = cur.fetchone()[0] - if row_count == 0: + has_rows = cur.fetchone()[0] + if not has_rows: raise AceException("Table filter produced no rows") except Exception as e: @@ -1126,6 +1309,7 @@ def table_diff_checks( td_task.fields.host_map = host_map td_task.fields.cols = cols td_task.fields.key = key + td_task.fields.simple_primary_key = len(key.split(",")) == 1 if td_task.fields.col_types and len(td_task.fields.col_types) > 1: ref_node = list(td_task.fields.col_types.keys())[0] @@ -1177,6 +1361,289 @@ def table_diff_checks( return td_task +def validate_merkle_tree_inputs( + mtree_task: MerkleTreeTask, + skip_table_check: bool = False, + override_block_size: bool = False, +) -> None: + """ + Validates the basic inputs for a merkle tree task without establishing connections. + Raises AceException if validation fails. + """ + if not mtree_task.cluster_name or mtree_task.cluster_name == "": + raise AceException("cluster_name is a required argument") + + if type(mtree_task.block_size) is str: + try: + mtree_task.block_size = int(mtree_task.block_size) + except Exception: + raise AceException("Invalid values for ACE_MTREE_BLOCK_SIZE") + elif type(mtree_task.block_size) is not int: + raise AceException("Invalid value type for ACE_MTREE_BLOCK_SIZE") + + if not override_block_size: + # Capping max block size here to prevent the hash function from taking forever + if mtree_task.block_size > config.MAX_MTREE_BLOCK_SIZE: + raise AceException(f"Block size should be <= {config.MAX_MTREE_BLOCK_SIZE}") + if mtree_task.block_size < config.MIN_MTREE_BLOCK_SIZE: + raise AceException(f"Block size should be >= {config.MIN_MTREE_BLOCK_SIZE}") + + if type(mtree_task.max_cpu_ratio) is int: + mtree_task.max_cpu_ratio = float(mtree_task.max_cpu_ratio) + elif type(mtree_task.max_cpu_ratio) is str: + try: + mtree_task.max_cpu_ratio = float(mtree_task.max_cpu_ratio) + except Exception: + raise AceException("Invalid values for ACE_MAX_CPU_RATIO") + elif type(mtree_task.max_cpu_ratio) is not float: + raise AceException("Invalid value type for ACE_MAX_CPU_RATIO") + + if mtree_task.max_cpu_ratio > 1.0 or mtree_task.max_cpu_ratio < 0.0: + raise AceException( + "Invalid value range for ACE_MAX_CPU_RATIO or --max_cpu_ratio" + ) + + mtree_task.rebalance = parse_bool_field("rebalance", mtree_task.rebalance) + mtree_task.recreate_objects = parse_bool_field( + "recreate_objects", mtree_task.recreate_objects + ) + mtree_task.write_ranges = parse_bool_field("write_ranges", mtree_task.write_ranges) + + if mtree_task.ranges_file: + if not os.path.exists(mtree_task.ranges_file): + raise AceException(f"File {mtree_task.ranges_file} does not exist") + + try: + block_ranges = ast.literal_eval(open(mtree_task.ranges_file, "r").read()) + if not block_ranges: + raise AceException( + f"Ranges file {mtree_task.ranges_file} is empty or invalid" + ) + for block_range in block_ranges: + if len(block_range) != 3: + raise AceException( + f"Range {block_range} must be of form (block_id, start, end)" + ) + mtree_task.ranges = block_ranges + except Exception as e: + raise AceException( + f"Error parsing ranges file {mtree_task.ranges_file}: {e}" + ) + + node_list = [] + try: + node_list = parse_nodes(mtree_task._nodes) + except ValueError as e: + raise AceException( + "Nodes should be a comma-separated list of nodenames " + + f'\n\tE.g., --nodes="n1,n2". Error: {e}' + ) + + if len(node_list) > 3: + raise AceException( + "mtree-diff currently supports up to a three-way table comparison" + ) + + found = check_cluster_exists(mtree_task.cluster_name) + if found: + util.message( + f"Cluster {mtree_task.cluster_name} exists", + p_state="success", + quiet_mode=mtree_task.quiet_mode, + ) + else: + raise AceException(f"Cluster {mtree_task.cluster_name} not found") + + if not skip_table_check: + if ( + type(mtree_task._table_name) is not str + or mtree_task._table_name.strip() == "" + ): + raise AceException("table_name is a required argument") + + nm_lst = mtree_task._table_name.split(".") + if len(nm_lst) != 2: + raise AceException( + f"TableName {mtree_task._table_name} must be of form" + " 'schema.table_name'" + ) + l_schema, l_table = nm_lst + + l_schema = sanitise_input(l_schema) + l_table = sanitise_input(l_table) + + db, pg, node_info = cluster.load_json(mtree_task.cluster_name) + + cluster_nodes = [] + database = {} + + if mtree_task._dbname: + for db_entry in db: + if db_entry["db_name"] == mtree_task._dbname: + database = db_entry + break + else: + database = db[0] + + if not database: + raise AceException( + f"Database '{mtree_task._dbname}' " + + f"not found in cluster '{mtree_task.cluster_name}'" + ) + + # Combine db and cluster_nodes into a single json + for node in node_info: + if node_list and node["name"] not in node_list: + continue + combined_json = {**database, **node} + cluster_nodes.append(combined_json) + + if not node_list: + node_list = [node["name"] for node in cluster_nodes] + + if mtree_task._nodes == "all" and len(cluster_nodes) > 3: + raise AceException("Merkle tree diff only supports up to three way comparison") + + if mtree_task._nodes != "all" and len(node_list) > 1: + for n in node_list: + if not any(filter(lambda x: x["name"] == n, cluster_nodes)): + raise AceException("Specified nodenames not present in cluster") + + # Store basic task information + mtree_task.fields.l_schema = l_schema if not skip_table_check else None + mtree_task.fields.l_table = l_table if not skip_table_check else None + mtree_task.fields.node_list = node_list + mtree_task.fields.database = database + mtree_task.fields.cluster_nodes = cluster_nodes + + +def merkle_tree_checks( + mtree_task: MerkleTreeTask, skip_validation: bool = False +) -> None: + """ + Checks if a table is 'diffable' for a merkle tree. + + TODO: This is a temporary function since all these checks are already a part + of the table-diff checks. Will clean this up eventually. + + Returns: + The validated and prepared task + """ + + # First do basic validation + if not skip_validation: + validate_merkle_tree_inputs(mtree_task) + + # Now do connection-specific validation + cols = None + key = None + conn_params = [] + conn_list = [] + host_map = {} + required_privileges = ["SELECT"] + l_schema = mtree_task.fields.l_schema + l_table = mtree_task.fields.l_table + + try: + for node_info in mtree_task.fields.cluster_nodes: + hostname = node_info["name"] + host_ip = node_info["public_ip"] + user = node_info["db_user"] + port = node_info.get("port", 5432) + + if node_info["name"] in mtree_task.fields.node_list: + params, conn = mtree_task.connection_pool.get_cluster_node_connection( + node_info, + client_role=( + mtree_task.client_role + if mtree_task.invoke_method == "api" + else None + ), + ) + ensure_pgcrypto_installed(conn) + else: + continue + + curr_cols = get_cols(conn, l_schema, l_table) + curr_key = get_key(conn, l_schema, l_table) + + if not curr_cols: + raise AceException( + f"Table '{mtree_task._table_name}' not found on {hostname}" + ", or the current user does not have adequate privileges" + ) + if not curr_key: + raise AceException( + f"No primary key found for '{mtree_task._table_name}'" + ) + + if (not cols) and (not key): + cols = curr_cols + key = curr_key + + if (curr_cols != cols) or (curr_key != key): + raise AceException("Table schemas don't match") + + cols = curr_cols + key = curr_key + + col_types = get_col_types(conn, l_table) + col_types_key = f"{host_ip}:{port}" + + if not mtree_task.fields.col_types: + mtree_task.fields.col_types = {} + + mtree_task.fields.col_types[col_types_key] = col_types + + authorised, missing_privileges = check_user_privileges( + conn, + user, + l_schema, + l_table, + required_privileges, + ) + + # Missing privileges come back as table_, but we use + # "CREATE/SELECT/INSERT/UPDATE/DELETE" in the required_privileges list + # So, we're simply formatting it correctly here for the exception + # message + missing_privs = [ + m.split("_")[1].upper() + for m in missing_privileges + if m.split("_")[1].upper() in required_privileges + ] + exception_msg = ( + f'User "{user}" does not have the necessary privileges' + f" to run {', '.join(missing_privs)} " + f'on table "{l_schema}.{l_table}" ' + f'on node "{hostname}"' + ) + + if not authorised: + raise AceException(exception_msg) + + conn_list.append(conn) + conn_params.append(params) + host_map[host_ip + ":" + str(port)] = hostname + + except Exception as e: + raise e + + util.message( + "Connections successful to nodes in cluster", + p_state="success", + quiet_mode=mtree_task.quiet_mode, + ) + + # Psycopg connection objects cannot be pickled easily, + # so, we send the connection parameters instead + mtree_task.fields.conn_params = conn_params + mtree_task.fields.host_map = host_map + mtree_task.fields.cols = cols + mtree_task.fields.key = key + mtree_task.fields.simple_primary_key = len(key.split(",")) == 1 + + def check_repair_option_compatibility(tr_task: TableRepairTask) -> None: """ Checks if the repair options specified are compatible. @@ -1217,10 +1684,10 @@ def validate_table_repair_inputs(tr_task: TableRepairTask) -> None: Validates the basic inputs for a table repair task without establishing connections. Raises AceException if validation fails. """ - if not tr_task.cluster_name: + if type(tr_task.cluster_name) is not str or tr_task.cluster_name.strip() == "": raise AceException("cluster_name is a required argument") - if not tr_task.diff_file_path: + if type(tr_task.diff_file_path) is not str or tr_task.diff_file_path.strip() == "": raise AceException("diff_file is a required argument") tr_task.fix_nulls = parse_bool_field("fix_nulls", tr_task.fix_nulls) @@ -1263,8 +1730,10 @@ def validate_table_repair_inputs(tr_task: TableRepairTask) -> None: f"TableName {tr_task._table_name} must be of form" "'schema.table_name'" ) - l_schema = nm_lst[0] - l_table = nm_lst[1] + l_schema, l_table = nm_lst + + l_schema = sanitise_input(l_schema) + l_table = sanitise_input(l_table) db, pg, node_info = cluster.load_json(tr_task.cluster_name) @@ -1355,8 +1824,6 @@ def table_repair_checks( params, conn = tr_task.connection_pool.get_cluster_node_connection( nd, - tr_task.cluster_name, - invoke_method=tr_task.invoke_method, client_role=(tr_task.client_role if config.USE_CERT_AUTH else None), ) @@ -1436,23 +1903,25 @@ def validate_repset_diff_inputs(rd_task: RepsetDiffTask) -> None: Validates the basic inputs for a repset diff task without establishing connections. Raises AceException if validation fails. """ - if type(rd_task.block_rows) is str: + if type(rd_task.cluster_name) is not str or rd_task.cluster_name.strip() == "": + raise AceException("cluster_name is a required argument") + + if type(rd_task.repset_name) is not str or rd_task.repset_name.strip() == "": + raise AceException("repset_name is a required argument") + + if type(rd_task.block_size) is str: try: - rd_task.block_rows = int(rd_task.block_rows) + rd_task.block_size = int(rd_task.block_size) except Exception: - raise AceException("Invalid values for ACE_BLOCK_ROWS or --block_rows") - elif type(rd_task.block_rows) is not int: - raise AceException("Invalid value type for ACE_BLOCK_ROWS or --block_rows") + raise AceException("Invalid values for DIFF_BLOCK_SIZE or --block_size") + elif type(rd_task.block_size) is not int: + raise AceException("Invalid value type for DIFF_BLOCK_SIZE or --block_size") # Capping max block size here to prevent the hash function from taking forever - if rd_task.block_rows > config.MAX_ALLOWED_BLOCK_SIZE: - raise AceException( - f"Block row size should be <= {config.MAX_ALLOWED_BLOCK_SIZE}" - ) - if rd_task.block_rows < config.MIN_ALLOWED_BLOCK_SIZE: - raise AceException( - f"Block row size should be >= {config.MIN_ALLOWED_BLOCK_SIZE}" - ) + if rd_task.block_size > config.MAX_DIFF_BLOCK_SIZE: + raise AceException(f"Block row size should be <= {config.MAX_DIFF_BLOCK_SIZE}") + if rd_task.block_size < config.MIN_DIFF_BLOCK_SIZE: + raise AceException(f"Block row size should be >= {config.MIN_DIFF_BLOCK_SIZE}") if type(rd_task.max_cpu_ratio) is int: rd_task.max_cpu_ratio = float(rd_task.max_cpu_ratio) @@ -1533,6 +2002,7 @@ def validate_repset_diff_inputs(rd_task: RepsetDiffTask) -> None: if not any(filter(lambda x: x["name"] == n, cluster_nodes)): raise AceException("Specified nodenames not present in cluster") + rd_task.repset_name = sanitise_input(rd_task.repset_name) rd_task.fields.cluster_nodes = cluster_nodes rd_task.fields.database = database rd_task.fields.node_list = node_list @@ -1567,10 +2037,9 @@ def repset_diff_checks( ) or (not rd_task.fields.node_list): _, conn = rd_task.connection_pool.get_cluster_node_connection( nd, - rd_task.cluster_name, - invoke_method=rd_task.invoke_method, client_role=(rd_task.client_role if config.USE_CERT_AUTH else None), ) + ensure_pgcrypto_installed(conn) conn_list.append(conn) except Exception as e: @@ -1608,9 +2077,7 @@ def repset_diff_checks( ) # Convert fetched rows into a list of strings - rd_task.table_list = [ - table[0] for table in tables if table[0] not in rd_task.skip_tables - ] + rd_task.table_list = [table[0] for table in tables] return rd_task @@ -1708,8 +2175,6 @@ def spock_diff_checks( ) or (not sd_task.fields.node_list): params, conn = sd_task.connection_pool.get_cluster_node_connection( nd, - sd_task.cluster_name, - invoke_method=sd_task.invoke_method, client_role=(sd_task.client_role if config.USE_CERT_AUTH else None), ) conn_params.append(params) @@ -1726,6 +2191,14 @@ def spock_diff_checks( def validate_schema_diff_inputs(sc_task: SchemaDiffTask) -> SchemaDiffTask: + if type(sc_task.cluster_name) is not str or sc_task.cluster_name.strip() == "": + raise AceException("cluster_name is a required argument") + + if type(sc_task.schema_name) is not str or sc_task.schema_name.strip() == "": + raise AceException("schema_name is a required argument") + + sc_task.ddl_only = parse_bool_field("ddl_only", sc_task.ddl_only) + node_list = [] try: node_list = parse_nodes(sc_task._nodes) @@ -1819,10 +2292,12 @@ def schema_diff_checks( ) or (not sc_task.fields.node_list): _, conn = sc_task.connection_pool.get_cluster_node_connection( nd, - sc_task.cluster_name, - invoke_method=sc_task.invoke_method, client_role=(sc_task.client_role if config.USE_CERT_AUTH else None), ) + + if not sc_task.ddl_only: + ensure_pgcrypto_installed(conn) + conn_list.append(conn) except Exception as e: @@ -1955,7 +2430,7 @@ def update_spock_exception_checks( for node in cluster_nodes: if node["name"] == node_name: # FIXME: Figure out connection handling here - _, conn = conn_pool.get_cluster_node_connection(node, cluster_name) + _, conn = conn_pool.get_cluster_node_connection(node) conn.autocommit = False except Exception as e: @@ -2003,8 +2478,6 @@ def handle_task_exception(task, task_context): } _, conn = task.connection_pool.get_cluster_node_connection( node_info, - task.cluster_name, - invoke_method=task.invoke_method, client_role=( task.client_role if config.USE_CERT_AUTH and task.invoke_method == "api" @@ -2072,15 +2545,14 @@ def error_listener(event): ace_db.create_ace_tables() - fire.Fire( - { - "table-diff": ace_cli.table_diff_cli, - "table-repair": ace_cli.table_repair_cli, - "table-rerun": ace_cli.table_rerun_cli, - "repset-diff": ace_cli.repset_diff_cli, - "schema-diff": ace_cli.schema_diff_cli, - "spock-diff": ace_cli.spock_diff_cli, - "spock-exception-update": ace_cli.update_spock_exception_cli, - "start": ace_cli.start_cli, - } - ) + # Check if the last argument is --help or -h for mtree command + # If it is, we remove it to prevent fire.Fire from interpreting it + # This ensures that the mtree command properly renders help for + # ./pgedge ace mtree or ./pgedge ace mtree --help|-h + if sys.argv[-1] in ("--help", "-h") and "mtree" in sys.argv: + mtree_index = sys.argv.index("mtree") + received_args = sys.argv[mtree_index + 1 :] + if received_args == ["--help"] or received_args == ["-h"]: + sys.argv = sys.argv[:-1] + + fire.Fire(ace_cli.AceCLI()) diff --git a/cli/scripts/ace_auth.py b/cli/scripts/ace_auth.py index 8e87a01f..7ab1803a 100644 --- a/cli/scripts/ace_auth.py +++ b/cli/scripts/ace_auth.py @@ -1,81 +1,91 @@ +from copy import deepcopy import datetime import ssl from functools import wraps from flask import request, jsonify import psycopg -import pgpasslib from cryptography.hazmat.backends import default_backend -from cryptography import x509 +from cryptography import x509, __version__ as crypto_version import ace_config as config from ace_exceptions import CertificateVerificationError, AuthenticationError +import pgpasslib -# TODO: Add this back once cloud fixes their dependencies -# if config.USE_NAIVE_DATETIME: -# from packaging import version -# # not_valid_before is deprecated from this version on -# # we will use the _utc equivalents unless we're dealing with an older version -# # https://cryptography.io/en/latest/x509/reference/#cryptography.x509.Certificate.not_valid_before_utc -# CRYPTO_VERSION_WITH_UTC = version.parse("42.0.0") -# USE_UTC_SUFFIX = version.parse(crypto_version) >= CRYPTO_VERSION_WITH_UTC -# else: -# # If you're using a version of cryptography that doesn't support the timezone -# # aware datetime objects, and USE_NAIVE_DATETIME is still set to False, -# # this will break. -# USE_UTC_SUFFIX = True +if config.USE_NAIVE_DATETIME: + from packaging import version + # not_valid_before is deprecated from this version on + # we will use the _utc equivalents unless we're dealing with an older version + # https://cryptography.io/en/latest/x509/reference/#cryptography.x509.Certificate.not_valid_before_utc + CRYPTO_VERSION_WITH_UTC = version.parse("42.0.0") + USE_UTC_SUFFIX = version.parse(crypto_version) >= CRYPTO_VERSION_WITH_UTC +else: + # If you're using a version of cryptography that doesn't support the timezone + # aware datetime objects, and USE_NAIVE_DATETIME is still set to False, + # this will break. + USE_UTC_SUFFIX = True USE_UTC_SUFFIX = True if not config.USE_NAIVE_DATETIME else False +# TODO: Get rid of passing around params class ConnectionPool: """Manages database connections to avoid creating duplicate connections.""" def __init__(self): - self._connections = {} # {(host, port, db_user, dbname): conn} - - def get_connection(self, host, port, db_user, dbname): - """ - Get an existing connection or return None if not found. - - Performs thorough connection check to ensure SSL connections are valid. - """ + self._connections = {} + self._params = { + "dbname": None, + "user": None, + "host": None, + "port": None, + "options": f"-c statement_timeout={config.STATEMENT_TIMEOUT}", + "application_name": "ACE", + "keepalives": 1, + "keepalives_idle": 30, + "keepalives_interval": 10, + "keepalives_count": 5, + "connect_timeout": config.CONNECTION_TIMEOUT, + } + + def get_conn_from_pool(self, host, port, db_user, dbname): + """Get an existing connection or return None if not found.""" key = (host, port, db_user, dbname) conn = self._connections.get(key) + params = deepcopy(self._params) + + params["dbname"] = dbname + params["user"] = db_user + params["host"] = host + params["port"] = port - # Check if connection is still alive and valid if conn: try: - # First check if connection is closed if conn.closed: del self._connections[key] - return None + return None, None - # Then check if connection is still valid with a transaction with conn.cursor() as cur: - # Start a transaction to ensure connection is truly alive - conn.rollback() # Clear any previous transaction state + conn.rollback() cur.execute("BEGIN") cur.execute("SELECT 1") cur.execute("COMMIT") # Reset role here. We'll handle privilege dropping later cur.execute("RESET ROLE") - return conn + return params, conn except Exception: - # If any error occurs, remove the connection if key in self._connections: try: conn.close() except Exception: pass del self._connections[key] - return None - return None + return None, None + return None, None - def add_connection(self, host, port, db_user, dbname, conn): + def add_conn_to_pool(self, host, port, db_user, dbname, conn): """Add a new connection to the pool.""" key = (host, port, db_user, dbname) - # Close existing connection if it exists if key in self._connections: try: self._connections[key].close() @@ -83,16 +93,11 @@ def add_connection(self, host, port, db_user, dbname, conn): pass self._connections[key] = conn - def get_cluster_node_connection( - self, - node_info, - cluster_name=None, - invoke_method="cli", - client_role=None, - drop_privileges=True, - ): + def connect(self, node_info): """ - Create a database connection to a cluster node with proper authentication. + Create a new independent connection to a node. + This is the central method for creating new PostgreSQL connections. + Handles both certificate and password authentication. Args: node_info: Dictionary containing node connection information: @@ -101,105 +106,83 @@ def get_cluster_node_connection( - public_ip: Host IP/name - port: Port number - db_password: Password (optional) - - name: Node name - cluster_name: Name of the cluster (for error messages) - invoke_method: Either "cli" or "api" to determine auth method - client_role: Role to switch to after connection (used in API mode) - Returns: - tuple: (connection_params, connection) - - Raises: - AuthenticationError: If authentication fails + psycopg.Connection: A new database connection """ try: - host = node_info["public_ip"] - port = node_info.get("port", 5432) - db_user = node_info["db_user"] - dbname = node_info["db_name"] - - params = { - "dbname": dbname, - "user": db_user, - "host": host, - "port": port, - "options": f"-c statement_timeout={config.STATEMENT_TIMEOUT}", - "application_name": "ACE", - "keepalives": 1, - "keepalives_idle": 30, - "keepalives_interval": 10, - "keepalives_count": 5, - "connect_timeout": config.CONNECTION_TIMEOUT, - } - - # Check connection pool first - conn = self.get_connection(host, port, db_user, dbname) - if conn: - # If we're in API mode we have to switch to the user's role - # for security reasons. However, if ACE is being invoked through - # the CLI, we don't have to worry about this. - if drop_privileges: - if config.USE_CERT_AUTH and client_role: - with conn.cursor() as cur: - cur.execute(f"SET ROLE {client_role}") - return params, conn + params = deepcopy(self._params) + params["dbname"] = node_info["db_name"] + params["user"] = node_info["db_user"] + params["host"] = node_info["public_ip"] + params["port"] = node_info.get("port", 5432) if config.USE_CERT_AUTH: - if config.ACE_USER_CERT_FILE and config.ACE_USER_KEY_FILE: - params.update( - { - "sslmode": "verify-full", - "sslcert": config.ACE_USER_CERT_FILE, - "sslkey": config.ACE_USER_KEY_FILE, - "sslrootcert": config.CA_CERT_FILE, - } - ) - else: - raise AuthenticationError( - "Client certificate authentication is enabled but no" - "certificate files are provided" - ) + params.update( + { + "sslmode": "verify-full", + "sslcert": config.ACE_USER_CERT_FILE, + "sslkey": config.ACE_USER_KEY_FILE, + "sslrootcert": config.CA_CERT_FILE, + } + ) + elif "db_password" in node_info: + params["password"] = node_info["db_password"] else: - if invoke_method == "api": - raise AuthenticationError( - "Client certificate authentication needs to be enabled" - "for API usage" - ) - # Handle password authentication - if node_info["db_password"]: - params["password"] = node_info["db_password"] - else: - pgpass = pgpasslib.getpass( - host=node_info["name"], - user=db_user, - dbname=dbname, - port=port, - ) - if not pgpass: - msg = f"No password found for {node_info['name']}" - if cluster_name: - msg += f" in {cluster_name}.json or ~/.pgpass" - raise AuthenticationError(msg) - params["password"] = pgpass + pgpass = pgpasslib.getpass( + host=node_info["public_ip"], + user=node_info["db_user"], + dbname=node_info["db_name"], + port=node_info.get("port", 5432), + ) + if not pgpass: + msg = f"No password found for {node_info['public_ip']}" + raise AuthenticationError(msg) + params["password"] = pgpass conn = psycopg.connect(**params) - # Here again, we need to switch to the user's role if we're in API mode - if drop_privileges: - if config.USE_CERT_AUTH and client_role: - with conn.cursor() as cur: - cur.execute(f"SET ROLE {client_role}") - - self.add_connection(host, port, db_user, dbname, conn) return params, conn - except AuthenticationError: - raise except Exception as e: raise AuthenticationError( f"Failed to connect to node {node_info['public_ip']}: {str(e)}" ) + def get_cluster_node_connection( + self, + node_info, + client_role=None, + drop_privileges=True, + ): + """ + Get a connection to a cluster node, using the connection pool if possible. + If no pooled connection exists, creates a new one using get_new_conn. + + Args: + node_info: Dictionary containing node connection information + cluster_name: Name of the cluster (for error messages) + invoke_method: Either "cli" or "api" to determine auth method + client_role: Role to switch to after connection (used in API mode) + drop_privileges: Whether to drop to client_role after connecting + + Returns: + tuple: (connection_params, connection) + """ + host = node_info["public_ip"] + port = node_info.get("port", 5432) + db_user = node_info["db_user"] + dbname = node_info["db_name"] + + params, conn = self.get_conn_from_pool(host, port, db_user, dbname) + if conn: + if drop_privileges: + self.drop_privileges(conn, client_role) + return params, conn + + params, conn = self.connect(node_info) + self.add_conn_to_pool(host, port, db_user, dbname, conn) + return params, conn + def drop_privileges(self, conn, client_role): if config.USE_CERT_AUTH and client_role: with conn.cursor() as cur: @@ -263,11 +246,9 @@ def verify_client_cert(cert_data, ca_cert_path=None): context.check_hostname = False cert = x509.load_pem_x509_certificate(cert_data, default_backend()) - # Verify certificate is properly formatted and not expired if not cert: raise CertificateVerificationError("Invalid certificate format") - # Use appropriate attribute names based on cryptography version not_valid_before = ( cert.not_valid_before_utc if USE_UTC_SUFFIX else cert.not_valid_before ) @@ -280,12 +261,8 @@ def verify_client_cert(cert_data, ca_cert_path=None): "Certificate has invalid validity period" ) - # Use timezone-aware datetime for comparison current_time = datetime.datetime.now(datetime.timezone.utc) - if ( - current_time < not_valid_before - or current_time > not_valid_after - ): + if current_time < not_valid_before or current_time > not_valid_after: raise CertificateVerificationError("Certificate is not currently valid") return True @@ -309,19 +286,15 @@ def protected_endpoint(): @wraps(f) def decorated_function(*args, **kwargs): try: - # Get client certificate from request environment if "SSL_CLIENT_CERT" not in request.environ: return jsonify({"error": "Client certificate required"}), 401 cert_data = request.environ["SSL_CLIENT_CERT"].encode("utf-8") - # Verify certificate using CA cert from config verify_client_cert(cert_data) - # Extract CN client_cn = extract_common_name(cert_data) - # Add CN to request context for use in the endpoint request.client_cn = client_cn return f(*args, **kwargs) diff --git a/cli/scripts/ace_cli.py b/cli/scripts/ace_cli.py index 16d43796..cb4ef764 100644 --- a/cli/scripts/ace_cli.py +++ b/cli/scripts/ace_cli.py @@ -11,542 +11,901 @@ SpockDiffTask, TableDiffTask, TableRepairTask, + MerkleTreeTask, ) import ace +import ace_mtree import util from ace_exceptions import AceException -def table_diff_cli( - cluster_name, - table_name, - dbname=None, - block_rows=config.BLOCK_ROWS_DEFAULT, - max_cpu_ratio=config.MAX_CPU_RATIO_DEFAULT, - output="json", - nodes="all", - batch_size=config.BATCH_SIZE_DEFAULT, - table_filter=None, - quiet=False, -): +class MerkleTreeCLI(object): """ - Compare a table across a cluster and produce a report showing - any differences. - - Args: - cluster_name (str): Name of the cluster where the operation should be performed. - table_name (str): Schema-qualified name of the table that you are - comparing across cluster nodes. - dbname (str, optional): Name of the database. Defaults to the name of - the first database in the cluster configuration. - block_rows (int, optional): Number of rows to process per block. - Defaults to config.BLOCK_ROWS_DEFAULT. - max_cpu_ratio (float, optional): Maximum CPU utilisation. The accepted - range is 0.0-1.0. Defaults to config.MAX_CPU_RATIO_DEFAULT. - output (str, optional): Output format. Acceptable values are "json", - "csv", and "html". Defaults to "json". - nodes (str, optional): Comma-delimited subset of nodes on which the - command will be executed. Defaults to "all". - batch_size (int, optional): Size of each batch. Defaults to - config.BATCH_SIZE_DEFAULT. - table_filter (str, optional): A SQL WHERE clause that allows you to - filter rows for comparison. - quiet (bool, optional): Whether to suppress output in stdout. Defaults - to False. - - Raises: - AceException: If there's an error specific to the ACE operation. - Exception: For any unexpected errors during the table diff operation. - - Returns: - None. The function performs the table diff operation and handles any - exceptions. All output messages are printed to stdout since it's a CLI - function. + Use pre-computed table hashes, maintained as Merkle Trees, to achieve a + significant speed up over normal-mode table-diff. """ - task_id = ace_db.generate_task_id() - try: - td_task = TableDiffTask( - cluster_name=cluster_name, - _table_name=table_name, - _dbname=dbname, - block_rows=block_rows, - max_cpu_ratio=max_cpu_ratio, - output=output, - _nodes=nodes, - batch_size=batch_size, - quiet_mode=quiet, - table_filter=table_filter, - invoke_method="cli", - ) - td_task.scheduler.task_id = task_id - td_task.scheduler.task_type = "table-diff" - td_task.scheduler.task_status = "RUNNING" - td_task.scheduler.started_at = datetime.now() - - ace.validate_table_diff_inputs(td_task) - ace_db.create_ace_task(task=td_task) - ace_core.table_diff(td_task) - td_task.connection_pool.close_all() - except AceException as e: - util.exit_message(str(e)) - except Exception as e: - traceback.print_exc() - util.exit_message(f"Unexpected error while running table diff: {e}") - - -def table_repair_cli( - cluster_name, - table_name, - diff_file, - source_of_truth=None, - dbname=None, - dry_run=False, - quiet=False, - generate_report=False, - insert_only=False, - upsert_only=False, - fix_nulls=False, - fire_triggers=False, - bidirectional=False, -): """ - Repair a table across a cluster by fixing data inconsistencies identified - in a table-diff operation. - - Args: - cluster_name (str): Name of the cluster where the operation should be performed. - diff_file (str): Path to the diff file generated by a previous table diff. - source_of_truth (str): Node name to be used as the source of truth for - the repair. - table_name (str): Schema-qualified name of the table that you are - comparing across cluster nodes. - dbname (str, optional): Name of the database. Defaults to the name of - the first database in the cluster configuration. - dry_run (bool, optional): If True, simulates the repair without making - changes. Defaults to False. - generate_report (bool, optional): If True, generates a detailed report - of the repair. Defaults to False. - upsert_only (bool, optional): If True, only performs upsert operations, - skipping deletions. Defaults to False. - insert_only (bool, optional): If True, only performs insert operations, - skipping updates and deletions. - fix_nulls (bool, optional): If True, fixes null values in the table - columns by looking at the corresponding column in the other nodes. - Does not need the source of truth to be specified. Must be used - only in special cases. This is not a recommended option for - repairing divergence. Defaults to False. - fire_triggers (bool, optional): If True, instructs triggers to fire - when a repair is performed; note that ENABLE ALWAYS triggers will - fire regardless of the value. - bidirectional (bool, optional): If True, performs a bidirectional - repair, applies differences found between nodes to create a - distinct union of the content. In a distinct union, each row that - is missing is recreated on the node from which it is missing, - eventually leading to a data set (on all nodes) in which all rows - are represented exactly once. - quiet (bool, optional): Whether to suppress output in stdout. Defaults - to False. - - Raises: - AceException: If there's an error specific to the ACE operation. - Exception: For any unexpected errors during the table repair operation. - - Returns: - None. The function performs the table repair operation and handles any - exceptions. All output messages are printed to stdout since it's a CLI - function. + Initialises the MerkleTreeCLI and sets up command groups. + This allows it to behave like another module under ace """ - task_id = ace_db.generate_task_id() - try: - tr_task = TableRepairTask( + def __init__(self): + self._commands = { + "init": self.init, + "build": self.build, + "update": self.update, + "table-diff": self.table_diff, + "teardown": self.teardown, + } + + def __getattr__(self, name): + try: + return self._commands[name] + except KeyError: + raise AttributeError(f"No such command: {name}") + + def __dir__(self): + # Allows Fire to generate proper helptext using dash-case commands + return list(self._commands.keys()) + + def _execute_task(self, mode, **kwargs): + """Helper to run merkle tree tasks.""" + task_id = ace_db.generate_task_id() + + try: + mtree_task = MerkleTreeTask( + mode=mode, + cluster_name=kwargs.get("cluster_name"), + _table_name=kwargs.get("table_name"), + _dbname=kwargs.get("dbname"), + analyse=kwargs.get("analyse", False), + rebalance=kwargs.get("rebalance", False), + recreate_objects=kwargs.get("recreate_objects", False), + block_size=kwargs.get("block_size", config.MTREE_BLOCK_SIZE), + max_cpu_ratio=kwargs.get("max_cpu_ratio", config.MAX_CPU_RATIO), + batch_size=kwargs.get("batch_size", 1), + output=kwargs.get("output", "json"), + quiet_mode=kwargs.get("quiet_mode", False), + write_ranges=kwargs.get("write_ranges", False), + ranges_file=kwargs.get("ranges_file"), + _nodes=kwargs.get("nodes", "all"), + invoke_method="cli", + ) + mtree_task.scheduler.task_id = task_id + mtree_task.scheduler.task_type = f"mtree-{mode}" + mtree_task.scheduler.task_status = "RUNNING" + mtree_task.scheduler.started_at = datetime.now() + + override_block_size = kwargs.get("override_block_size", False) + + if ((mode == "teardown") and (not kwargs.get("table_name"))) or ( + mode == "init" + ): + ace.validate_merkle_tree_inputs( + mtree_task, + skip_table_check=True, + override_block_size=override_block_size, + ) + else: + ace.validate_merkle_tree_inputs( + mtree_task, override_block_size=override_block_size + ) + + ace_db.create_ace_task(task=mtree_task) + + if mode == "init": + ace_mtree.mtree_init_helper(mtree_task) + elif mode == "build": + ace_mtree.build_mtree(mtree_task) + elif mode == "update": + ace_mtree.update_mtree(mtree_task) + elif mode == "table-diff": + ace_mtree.merkle_tree_diff(mtree_task) + elif mode == "teardown": + ace_mtree.mtree_teardown_helper(mtree_task) + + mtree_task.connection_pool.close_all() + except AceException as e: + util.exit_message(str(e)) + except Exception as e: + traceback.print_exc() + util.exit_message(f"Unexpected error while running merkle tree: {e}") + + def init(self, cluster_name, dbname=None, nodes="all", quiet_mode=False): + """ + Initialises the database with necessary objects for Merkle trees. + + Args: + cluster_name (str): Name of the cluster. + dbname (str, optional): Name of the database. + nodes (str, optional): Comma-separated subset of nodes. + quiet_mode (bool, optional): Suppress output. + """ + self._execute_task( + "init", cluster_name=cluster_name, - diff_file_path=diff_file, - source_of_truth=source_of_truth, - _table_name=table_name, - _dbname=dbname, - dry_run=dry_run, - quiet_mode=quiet, - generate_report=generate_report, - insert_only=insert_only, - upsert_only=upsert_only, - fix_nulls=fix_nulls, - fire_triggers=fire_triggers, - bidirectional=bidirectional, - invoke_method="cli", + dbname=dbname, + nodes=nodes, + quiet_mode=quiet_mode, ) - tr_task.scheduler.task_id = task_id - tr_task.scheduler.task_type = "table-repair" - tr_task.scheduler.task_status = "RUNNING" - tr_task.scheduler.started_at = datetime.now() - - ace.validate_table_repair_inputs(tr_task) - ace_db.create_ace_task(task=tr_task) - - if fix_nulls: - ace_core.table_repair_fix_nulls(tr_task) - elif bidirectional: - ace_core.table_repair_bidirectional(tr_task) - else: - ace_core.table_repair(tr_task) - - tr_task.connection_pool.close_all() - except AceException as e: - util.exit_message(str(e)) - except Exception as e: - traceback.print_exc() - util.exit_message(f"Unexpected error while running table repair: {e}") - - -def table_rerun_cli( - cluster_name, - diff_file, - table_name, - dbname=None, - quiet=False, - behavior="multiprocessing", -): - """ - Reruns a table diff operation based on a previous diff file. - - Args: - cluster_name (str): Name of the cluster where the operation should be performed. - diff_file (str): Path to the diff file from a previous table diff - operation. - table_name (str): Schema-qualified name of the table that you are - comparing across cluster nodes. - dbname (str, optional): Name of the database. Defaults to the name of - the first database in the cluster configuration. - behavior (str, optional): The rerun behavior, either "multiprocessing" - or "hostdb". "multiprocessing" uses parallel processing for faster - execution. "hostdb" uses the host database to create temporary - tables for faster comparisons. Defaults to "multiprocessing". - quiet (bool, optional): Whether to suppress output in stdout. Defaults - to False. - - Raises: - AceException: If there's an error specific to the ACE operation. - Exception: For any unexpected errors during the table rerun operation. - - Returns: - None. The function performs the table rerun operation and handles any - exceptions. All output messages are printed to stdout since it's a CLI - function. - """ - task_id = ace_db.generate_task_id() - try: - td_task = TableDiffTask( + def build( + self, + cluster_name, + table_name, + dbname=None, + analyse=False, + recreate_objects=False, + block_size=config.MTREE_BLOCK_SIZE, + max_cpu_ratio=config.MAX_CPU_RATIO, + write_ranges=False, + ranges_file=None, + nodes="all", + quiet_mode=False, + skip_block_size_check=False, + ): + """ + Builds a new Merkle tree for a table. + + Args: + cluster_name (str): Name of the cluster. + table_name (str): Schema-qualified table name. + dbname (str, optional): Name of the database. + analyse (bool, optional): Run ANALYZE on the table. + recreate_objects (bool, optional): Drop and recreate Merkle tree objects. + block_size (int, optional): Rows per leaf block. + max_cpu_ratio (float, optional): Max CPU for parallel operations. + write_ranges (bool, optional): Write block ranges to a JSON file. + ranges_file (str, optional): Path to a file with pre-computed ranges. + nodes (str, optional): Comma-separated subset of nodes. + quiet_mode (bool, optional): Suppress output. + skip_block_size_check (bool, optional): Skip block size check, and + potentially tolerate unsafe block sizes. Defaults to False. + """ + self._execute_task( + "build", cluster_name=cluster_name, - _table_name=table_name, - _dbname=dbname, - block_rows=config.BLOCK_ROWS_DEFAULT, - max_cpu_ratio=config.MAX_CPU_RATIO_DEFAULT, - output="json", - _nodes="all", - batch_size=config.BATCH_SIZE_DEFAULT, - table_filter=None, - quiet_mode=quiet, - diff_file_path=diff_file, - invoke_method="cli", + table_name=table_name, + dbname=dbname, + analyse=analyse, + recreate_objects=recreate_objects, + block_size=block_size, + max_cpu_ratio=max_cpu_ratio, + write_ranges=write_ranges, + ranges_file=ranges_file, + nodes=nodes, + quiet_mode=quiet_mode, + override_block_size=skip_block_size_check, ) - td_task.scheduler.task_id = task_id - td_task.scheduler.task_type = "table-rerun" - td_task.scheduler.task_status = "RUNNING" - td_task.scheduler.started_at = datetime.now() - - ace.validate_table_diff_inputs(td_task) - ace_db.create_ace_task(task=td_task) - except AceException as e: - util.exit_message(str(e)) - except Exception as e: - traceback.print_exc() - util.exit_message(f"Unexpected error while running table rerun: {e}") - - try: - if behavior == "multiprocessing": - ace_core.table_rerun_async(td_task) - elif behavior == "hostdb": - ace_core.table_rerun_temptable(td_task) - else: - util.exit_message(f"Invalid behavior: {behavior}") - - td_task.connection_pool.close_all() - except AceException as e: - util.exit_message(str(e)) - except Exception as e: - traceback.print_exc() - util.exit_message(f"Unexpected error while running table rerun: {e}") - - -def repset_diff_cli( - cluster_name, - repset_name, - dbname=None, - block_rows=config.BLOCK_ROWS_DEFAULT, - max_cpu_ratio=config.MAX_CPU_RATIO_DEFAULT, - output="json", - nodes="all", - batch_size=config.BATCH_SIZE_DEFAULT, - quiet=False, - skip_tables=None, - skip_file=None, -): - """ - Compare a repset across a cluster and produce a report showing - any differences. - - Args: - cluster_name (str): Name of the cluster where the operation should be performed. - repset_name (str): Name of the repset to compare across cluster nodes. - dbname (str, optional): Name of the database. Defaults to the name of - the first database in the cluster configuration. - block_rows (int, optional): Number of rows to process per block. - Defaults to config.BLOCK_ROWS_DEFAULT. - max_cpu_ratio (float, optional): Maximum CPU utilisation. The accepted - range is 0.0-1.0. Defaults to config.MAX_CPU_RATIO_DEFAULT. - output (str, optional): Output format. Acceptable values are "json", - "csv", and "html". Defaults to "json". - nodes (str, optional): Comma-delimited subset of nodes on which the - command will be executed. Defaults to "all". - batch_size (int, optional): Size of each batch. Defaults to - config.BATCH_SIZE_DEFAULT. - quiet (bool, optional): Whether to suppress output in stdout. Defaults - to False. - skip_tables (list, optional): Comma-deliminated list of tables to skip. - skip_file (str, optional): Path to a file containing a list of tables to skip. - - Raises: - AceException: If there's an error specific to the ACE operation. - Exception: For any unexpected errors during the repset diff operation. - - Returns: - None. The function performs the repset diff operation and handles any - exceptions. All output messages are printed to stdout since it's a CLI - function. - """ - task_id = ace_db.generate_task_id() - try: - rd_task = RepsetDiffTask( + def update( + self, + cluster_name, + table_name, + dbname=None, + rebalance=False, + max_cpu_ratio=config.MAX_CPU_RATIO, + nodes="all", + quiet_mode=False, + ): + """ + Updates an existing Merkle tree. + + Args: + cluster_name (str): Name of the cluster. + table_name (str): Schema-qualified table name. + dbname (str, optional): Name of the database. + rebalance (bool, optional): Trigger rebalancing of the tree. + max_cpu_ratio (float, optional): Max CPU for parallel operations. + nodes (str, optional): Comma-separated subset of nodes. + quiet_mode (bool, optional): Suppress output. + """ + self._execute_task( + "update", cluster_name=cluster_name, - _dbname=dbname, - repset_name=repset_name, - block_rows=block_rows, + table_name=table_name, + dbname=dbname, + rebalance=rebalance, max_cpu_ratio=max_cpu_ratio, - output=output, - _nodes=nodes, - batch_size=batch_size, - quiet_mode=quiet, - invoke_method="cli", - skip_tables=skip_tables, - skip_file=skip_file, + nodes=nodes, + quiet_mode=quiet_mode, ) - rd_task.scheduler.task_id = task_id - rd_task.scheduler.task_type = "repset-diff" - rd_task.scheduler.task_status = "RUNNING" - rd_task.scheduler.started_at = datetime.now() - - ace.validate_repset_diff_inputs(rd_task) - ace_db.create_ace_task(task=rd_task) - ace_core.multi_table_diff(rd_task) - - # TODO: Figure out a way to handle repset-level connection pooling - # This close_all() is redundant currently - rd_task.connection_pool.close_all() - except AceException as e: - util.exit_message(str(e)) - except Exception as e: - traceback.print_exc() - util.exit_message(f"Unexpected error while running repset diff: {e}") - - -def spock_diff_cli( - cluster_name, - dbname=None, - nodes="all", - quiet=False, -): - """ - Compare the spock metadata across a cluster and produce a report showing - any differences. - - Args: - cluster_name (str): Name of the cluster where the operation should be - performed. - dbname (str, optional): Name of the database. Defaults to the name of - the first database in the cluster configuration. - nodes (str, optional): Comma-delimited subset of nodes on which the - command will be executed. Defaults to "all". - quiet (bool, optional): Whether to suppress output in stdout. Defaults - to False. - - Raises: - AceException: If there's an error specific to the ACE operation. - Exception: For any unexpected errors during the spock diff operation. - - Returns: - None. The function performs the spock diff operation and handles any - exceptions. All output messages are printed to stdout since it's a CLI - function. - """ - task_id = ace_db.generate_task_id() - try: - sd_task = SpockDiffTask( + def table_diff( + self, + cluster_name, + table_name, + dbname=None, + rebalance=False, + max_cpu_ratio=config.MAX_CPU_RATIO, + batch_size=1, + nodes="all", + output="json", + quiet_mode=False, + ): + """ + Compares Merkle trees of a table across cluster nodes. + + Args: + cluster_name (str): Name of the cluster. + table_name (str): Schema-qualified table name. + dbname (str, optional): Name of the database. + rebalance (bool, optional): Trigger rebalancing of the tree. + max_cpu_ratio (float, optional): Max CPU for parallel operations. + batch_size (int, optional): Number of blocks per worker batch. + nodes (str, optional): Comma-separated subset of nodes. + output (str, optional): Output format (json, csv, html). + quiet_mode (bool, optional): Suppress output. + """ + self._execute_task( + "table-diff", cluster_name=cluster_name, - _dbname=dbname, - _nodes=nodes, - quiet_mode=quiet, - invoke_method="cli", + table_name=table_name, + dbname=dbname, + rebalance=rebalance, + max_cpu_ratio=max_cpu_ratio, + batch_size=batch_size, + nodes=nodes, + output=output, + quiet_mode=quiet_mode, ) - sd_task.scheduler.task_id = task_id - sd_task.scheduler.task_type = "spock-diff" - sd_task.scheduler.task_status = "RUNNING" - sd_task.scheduler.started_at = datetime.now() - - ace.validate_spock_diff_inputs(sd_task) - ace_db.create_ace_task(task=sd_task) - ace_core.spock_diff(sd_task) - sd_task.connection_pool.close_all() - except AceException as e: - util.exit_message(str(e)) - except Exception as e: - traceback.print_exc() - util.exit_message(f"Unexpected error while running spock diff: {e}") - - -def schema_diff_cli( - cluster_name, - schema_name, - nodes="all", - dbname=None, - ddl_only=True, - skip_tables=None, - skip_file=None, - quiet=False, -): - """ - Compare a schema across a cluster and produce a report showing - any differences. - - Args: - cluster_name (str): Name of the cluster where the operation should - be performed. - schema_name (str): Name of the schema that you are comparing across - cluster nodes. - nodes (str, optional): Comma-delimited subset of nodes on which the - command will be executed. Defaults to "all". - dbname (str, optional): Name of the database. Defaults to the name of - the first database in the cluster configuration. - ddl_only (bool, optional): If True, only compares DDL differences - across nodes. - skip_tables (list, optional): Comma-delimited list of tables to skip. - skip_file (str, optional): Path to a file containing a list of tables - to skip. - quiet (bool, optional): Whether to suppress output in stdout. Defaults - to False. - - Raises: - AceException: If there's an error specific to the ACE operation. - Exception: For any unexpected errors during the schema diff operation. - - Returns: - None. The function performs the schema diff operation and handles any - exceptions. All output messages are printed to stdout since it's a CLI - function. - """ - task_id = ace_db.generate_task_id() - try: - sc_task = SchemaDiffTask( + def teardown( + self, cluster_name, table_name=None, dbname=None, nodes="all", quiet_mode=False + ): + """ + Removes Merkle tree objects. + + Args: + cluster_name (str): Name of the cluster. + table_name (str, optional): Schema-qualified table name. If omitted, + removes objects for the entire database. + dbname (str, optional): Name of the database. + nodes (str, optional): Comma-separated subset of nodes. + quiet_mode (bool, optional): Suppress output. + """ + self._execute_task( + "teardown", cluster_name=cluster_name, - schema_name=schema_name, - _dbname=dbname, - _nodes=nodes, - ddl_only=ddl_only, - skip_tables=skip_tables, - skip_file=skip_file, - quiet_mode=quiet, - invoke_method="cli", + table_name=table_name, + dbname=dbname, + nodes=nodes, + quiet_mode=quiet_mode, ) - sc_task.scheduler.task_id = task_id - sc_task.scheduler.task_type = "schema-diff" - sc_task.scheduler.task_status = "RUNNING" - sc_task.scheduler.started_at = datetime.now() - - ace.schema_diff_checks(sc_task) - ace_db.create_ace_task(task=sc_task) - if ddl_only: - ace_core.schema_diff_objects(sc_task) - else: - ace_core.multi_table_diff(sc_task) - sc_task.connection_pool.close_all() - except AceException as e: - util.exit_message(str(e)) - except Exception as e: - traceback.print_exc() - util.exit_message(f"Unexpected error while running schema diff: {e}") - - -def update_spock_exception_cli(cluster_name, node_name, entry, dbname=None) -> None: - """ - Updates the Spock exception status for a specified cluster and node. - - Args: - cluster_name (str): Name of the cluster where the operation should - be performed. - node_name (str): The name of the node within the cluster where the - update should be performed. - entry (str): A JSON string representing the exception entry. The JSON object - parsed from this string should contain the following keys: - - "remote_origin" (str): Identifier of the origin node of the - transaction that caused the exception. (Required) - - "remote_commit_ts" (str): Commit timestamp of the - transaction on the remote origin. (Required) - - "remote_xid" (str): Transaction ID on the remote origin. - (Required) - - "status" (str): The new status to set for the exception (e.g., - "RESOLVED", "IGNORED"). (Required) - - "resolution_details" (dict, optional): A JSON serialisable dictionary - containing details about the resolution. - - "command_counter" (int, optional): If specified, only the specific - exception detail (matching this command_counter along with - remote_origin, remote_commit_ts, remote_xid) in the - `spock.exception_status_detail` table is updated. If omitted, - the main entry in `spock.exception_status` and all related - detail entries for the (remote_origin, remote_commit_ts, - remote_xid) trio in `spock.exception_status_detail` are updated. - dbname (str, optional): Name of the database. Defaults to the name of - the first database in the cluster configuration. - - Raises: - AceException: If an error specific to the ACE system occurs. - json.JSONDecodeError: If the provided exception entry is not valid - JSON. - Exception: For any other unexpected errors. - - Returns: - None - """ - - try: - conn = ace.update_spock_exception_checks(cluster_name, node_name, entry, dbname) - ace_core.update_spock_exception(entry, conn) - except AceException as e: - util.exit_message(str(e)) - except json.JSONDecodeError: - util.exit_message("Exception entry is not a valid JSON") - except Exception as e: - traceback.print_exc() - util.exit_message(f"Unexpected error while running exception status: {e}") - util.message("Spock exception status updated successfully", p_state="success") - -def start_cli() -> None: +class TableDiffCLI(object): + + def __init__(self): + pass + + def run( + self, + cluster_name, + table_name, + dbname=None, + block_rows=None, + max_cpu_ratio=config.MAX_CPU_RATIO, + output="json", + nodes="all", + batch_size=config.DIFF_BATCH_SIZE, + table_filter=None, + quiet=False, + skip_block_size_check=False, + **kwargs, + ): + """ + Compare a table across a cluster and produce a report showing + differences, if any. + + Args: + cluster_name (str): Name of the cluster where the operation should be + performed. + table_name (str): Schema-qualified name of the table that you are + comparing across cluster nodes. + dbname (str, optional): Name of the database to use. If omitted, + defaults to the first database in the cluster configuration file. + block_rows (int, optional): Number of rows to process per block. + Defaults to config.DIFF_BLOCK_SIZE. + max_cpu_ratio (float, optional): Maximum CPU utilisation. The accepted + range is 0.0-1.0. Defaults to config.MAX_CPU_RATIO_DEFAULT. + output (str, optional): Output format. Acceptable values are "json", + "csv", and "html". Defaults to "json". + nodes (str, optional): Comma-separated subset of nodes on which the + command will be executed. Defaults to "all". + batch_size (int, optional): Size of each batch, i.e., number of blocks + each worker should process. Defaults to config.DIFF_BATCH_SIZE. + table_filter (str, optional): Used to compare a subset of rows in the table. + Specified as a WHERE clause of a SQL query. E.g., + --table-filter="customer_id < 100" will compare only rows with + customer_id less than 100. If omitted, the entire table is compared. + quiet (bool, optional): Whether to suppress output in stdout. Defaults + to False. + skip_block_size_check (bool, optional): Skip block size check, and + potentially tolerate unsafe block sizes. Defaults to False. + + Raises: + AceException: If there's an error specific to the ACE operation. + Exception: For any unexpected errors during the table diff operation. + + Returns: + None. The function performs the table diff operation and handles any + exceptions. All output messages are printed to stdout since it's a CLI + function. + """ + task_id = ace_db.generate_task_id() + + try: + block_size = ( + kwargs.get( + "block_rows", kwargs.get("block_size", config.DIFF_BLOCK_SIZE) + ) + if not block_rows + else block_rows + ) + + td_task = TableDiffTask( + cluster_name=cluster_name, + _table_name=table_name, + _dbname=dbname, + block_size=block_size, + max_cpu_ratio=max_cpu_ratio, + output=output, + _nodes=nodes, + batch_size=batch_size, + quiet_mode=quiet, + table_filter=table_filter, + invoke_method="cli", + _override_block_size=skip_block_size_check, + ) + td_task.scheduler.task_id = task_id + td_task.scheduler.task_type = "table-diff" + td_task.scheduler.task_status = "RUNNING" + td_task.scheduler.started_at = datetime.now() + + ace.validate_table_diff_inputs(td_task) + ace_db.create_ace_task(task=td_task) + ace_core.table_diff(td_task) + td_task.connection_pool.close_all() + except AceException as e: + util.exit_message(str(e)) + except Exception as e: + traceback.print_exc() + util.exit_message(f"Unexpected error while running table diff: {e}") + + +class TableRepairCLI(object): + + def __init__(self): + pass + + def run( + self, + cluster_name, + table_name, + diff_file, + source_of_truth=None, + dbname=None, + dry_run=False, + quiet=False, + generate_report=False, + insert_only=False, + upsert_only=False, + fix_nulls=False, + fire_triggers=False, + bidirectional=False, + ): + """ + Repair a table across a cluster by fixing data inconsistencies identified + in a table-diff operation. + + Args: + cluster_name (str): Name of the cluster where the operation should be + performed. + diff_file (str): Path to the diff file generated by a previous table diff. + source_of_truth (str): Node name to be used as the source of truth for + the repair. + table_name (str): Schema-qualified name of the table that you are + comparing across cluster nodes. + dbname (str, optional): Name of the database. Defaults to the name of + the first database in the cluster configuration. + dry_run (bool, optional): If True, simulates the repair without making + changes. Defaults to False. + generate_report (bool, optional): If True, generates a detailed report + of the repair. Defaults to False. + upsert_only (bool, optional): If True, only performs upsert operations, + skipping deletions. Defaults to False. + insert_only (bool, optional): If True, only performs insert operations, + skipping updates and deletions. + fix_nulls (bool, optional): If True, fixes null values in the table + columns by looking at the corresponding column in the other nodes. + Does not need the source of truth to be specified. Must be used + only in special cases. This is not a recommended option for + repairing divergence. Defaults to False. + fire_triggers (bool, optional): If True, fires triggers on a table, if any, + during the repair process. Note that ENABLE ALWAYS triggers will fire + regardless of the value. + bidirectional (bool, optional): If True, performs a bidirectional + repair, applies differences found between nodes to create a + distinct union of the content. In a distinct union, each row that + is missing is recreated on the node from which it is missing, + eventually leading to a data set (on all nodes) in which all rows + are represented exactly once. + quiet (bool, optional): Whether to suppress output in stdout. Defaults + to False. + + Raises: + AceException: If there's an error specific to the ACE operation. + Exception: For any unexpected errors during the table repair operation. + + Returns: + None. The function performs the table repair operation and handles any + exceptions. All output messages are printed to stdout since it's a CLI + function. + """ + task_id = ace_db.generate_task_id() + + try: + tr_task = TableRepairTask( + cluster_name=cluster_name, + diff_file_path=diff_file, + source_of_truth=source_of_truth, + _table_name=table_name, + _dbname=dbname, + dry_run=dry_run, + quiet_mode=quiet, + generate_report=generate_report, + insert_only=insert_only, + upsert_only=upsert_only, + fix_nulls=fix_nulls, + fire_triggers=fire_triggers, + bidirectional=bidirectional, + invoke_method="cli", + ) + tr_task.scheduler.task_id = task_id + tr_task.scheduler.task_type = "table-repair" + tr_task.scheduler.task_status = "RUNNING" + tr_task.scheduler.started_at = datetime.now() + + ace.validate_table_repair_inputs(tr_task) + ace_db.create_ace_task(task=tr_task) + + if fix_nulls: + ace_core.table_repair_fix_nulls(tr_task) + elif bidirectional: + ace_core.table_repair_bidirectional(tr_task) + else: + ace_core.table_repair(tr_task) + + tr_task.connection_pool.close_all() + except AceException as e: + util.exit_message(str(e)) + except Exception as e: + traceback.print_exc() + util.exit_message(f"Unexpected error while running table repair: {e}") + + +class TableRerunCLI(object): + + def __init__(self): + pass + + def run( + self, + cluster_name, + diff_file, + table_name, + dbname=None, + behavior="rerun", + quiet=False, + ): + """ + Rerun a table diff operation based on a previous diff file. + + Args: + cluster_name (str): Name of the cluster where the operation should be + performed. + diff_file (str): Path to the diff file from a previous table diff + operation. + table_name (str): Schema-qualified name of the table that you are + comparing across cluster nodes. + dbname (str, optional): Name of the database to use. If omitted, + defaults to the first database in the cluster configuration. + behavior (str, optional, deprecated): Deprecated. Formerly used to + specify the behavior of the rerun. Now, it always defaults to "hostdb". + quiet (bool, optional): Whether to suppress output in stdout. Defaults + to False. + + Raises: + AceException: If there's an error specific to the ACE operation. + Exception: For any unexpected errors during the table rerun operation. + + Returns: + None. The function performs the table rerun operation and handles any + exceptions. All output messages are printed to stdout since it's a CLI + function. + """ + task_id = ace_db.generate_task_id() + + try: + td_task = TableDiffTask( + cluster_name=cluster_name, + _table_name=table_name, + _dbname=dbname, + block_size=config.DIFF_BLOCK_SIZE, + max_cpu_ratio=config.MAX_CPU_RATIO, + output="json", + _nodes="all", + batch_size=config.DIFF_BATCH_SIZE, + table_filter=None, + quiet_mode=quiet, + diff_file_path=diff_file, + invoke_method="cli", + ) + td_task.scheduler.task_id = task_id + td_task.scheduler.task_type = "table-rerun" + td_task.scheduler.task_status = "RUNNING" + td_task.scheduler.started_at = datetime.now() + + ace.validate_table_diff_inputs(td_task) + ace_db.create_ace_task(task=td_task) + except AceException as e: + util.exit_message(str(e)) + except Exception as e: + traceback.print_exc() + util.exit_message(f"Unexpected error while running table rerun: {e}") + + try: + ace_core.table_rerun_temptable(td_task) + td_task.connection_pool.close_all() + except AceException as e: + util.exit_message(str(e)) + except Exception as e: + traceback.print_exc() + util.exit_message(f"Unexpected error while running table rerun: {e}") + + +class RepsetDiffCLI(object): + + def __init__(self): + pass + + def run( + self, + cluster_name, + repset_name, + dbname=None, + block_size=config.DIFF_BLOCK_SIZE, + max_cpu_ratio=config.MAX_CPU_RATIO, + output="json", + nodes="all", + batch_size=config.DIFF_BATCH_SIZE, + quiet=False, + skip_tables=None, + skip_file=None, + ): + """ + Compare a repset across a cluster and produce a report showing + any differences. + + Args: + cluster_name (str): Name of the cluster where the operation should be + performed. + repset_name (str): Name of the repset to compare across cluster nodes. + dbname (str, optional): Name of the database to use. If omitted, + defaults to the first database in the cluster configuration. + block_size (int, optional): Number of rows to process per block. + Defaults to config.DIFF_BLOCK_SIZE. + max_cpu_ratio (float, optional): Maximum CPU utilisation. The accepted + range is 0.0-1.0. Defaults to config.MAX_CPU_RATIO_DEFAULT. + output (str, optional): Output format. Acceptable values are "json", + "csv", and "html". Defaults to "json". + nodes (str, optional): Comma-separated subset of nodes on which the + command will be executed. Defaults to "all". + batch_size (int, optional): Size of each batch, i.e., number of blocks + each worker should process. Defaults to config.DIFF_BATCH_SIZE. + quiet (bool, optional): Whether to suppress output in stdout. Defaults + to False. + skip_tables (list, optional): Comma-separated list of tables to skip. + If omitted, no tables are skipped. + skip_file (str, optional): Path to a file containing a list of tables to + skip. If omitted, no tables are skipped. + + Raises: + AceException: If there's an error specific to the ACE operation. + Exception: For any unexpected errors during the repset diff operation. + + Returns: + None. The function performs the repset diff operation and handles any + exceptions. All output messages are printed to stdout since it's a CLI + function. + """ + task_id = ace_db.generate_task_id() + + try: + rd_task = RepsetDiffTask( + cluster_name=cluster_name, + _dbname=dbname, + repset_name=repset_name, + block_size=block_size, + max_cpu_ratio=max_cpu_ratio, + output=output, + _nodes=nodes, + batch_size=batch_size, + quiet_mode=quiet, + invoke_method="cli", + skip_tables=skip_tables, + skip_file=skip_file, + ) + rd_task.scheduler.task_id = task_id + rd_task.scheduler.task_type = "repset-diff" + rd_task.scheduler.task_status = "RUNNING" + rd_task.scheduler.started_at = datetime.now() + + ace.validate_repset_diff_inputs(rd_task) + ace_db.create_ace_task(task=rd_task) + ace_core.multi_table_diff(rd_task) + + # TODO: Figure out a way to handle repset-level connection pooling + # This close_all() is redundant currently + rd_task.connection_pool.close_all() + except AceException as e: + util.exit_message(str(e)) + except Exception as e: + traceback.print_exc() + util.exit_message(f"Unexpected error while running repset diff: {e}") + + +class SchemaDiffCLI(object): + + def __init__(self): + pass + + def run( + self, + cluster_name, + schema_name, + nodes="all", + dbname=None, + ddl_only=True, + skip_tables=None, + skip_file=None, + quiet=False, + ): + """ + Compare a schema across a cluster and produce a report showing + any differences. + + Args: + cluster_name (str): Name of the cluster where the operation should + be performed. + schema_name (str): Name of the schema that you are comparing across + cluster nodes. + nodes (str, optional): Comma-delimited subset of nodes on which the + command will be executed. Defaults to "all". + dbname (str, optional): Name of the database. Defaults to the name of + the first database in the cluster configuration. + ddl_only (bool, optional): If True, only compares DDL differences + across nodes. + skip_tables (list, optional): Comma-delimited list of tables to skip. + skip_file (str, optional): Path to a file containing a list of tables + to skip. + quiet (bool, optional): Whether to suppress output in stdout. Defaults + to False. + + Raises: + AceException: If there's an error specific to the ACE operation. + Exception: For any unexpected errors during the schema diff operation. + + Returns: + None. The function performs the schema diff operation and handles any + exceptions. All output messages are printed to stdout since it's a CLI + function. + """ + task_id = ace_db.generate_task_id() + + try: + sc_task = SchemaDiffTask( + cluster_name=cluster_name, + schema_name=schema_name, + _dbname=dbname, + _nodes=nodes, + ddl_only=ddl_only, + skip_tables=skip_tables, + skip_file=skip_file, + quiet_mode=quiet, + invoke_method="cli", + ) + sc_task.scheduler.task_id = task_id + sc_task.scheduler.task_type = "schema-diff" + sc_task.scheduler.task_status = "RUNNING" + sc_task.scheduler.started_at = datetime.now() + + ace.schema_diff_checks(sc_task) + ace_db.create_ace_task(task=sc_task) + if sc_task.ddl_only: + ace_core.schema_diff_objects(sc_task) + else: + ace_core.multi_table_diff(sc_task) + sc_task.connection_pool.close_all() + except AceException as e: + util.exit_message(str(e)) + except Exception as e: + traceback.print_exc() + util.exit_message(f"Unexpected error while running schema diff: {e}") + + +class SpockDiffCLI(object): + + def __init__(self): + pass + + def run( + self, + cluster_name, + dbname=None, + nodes="all", + quiet=False, + ): + """ + Compare the spock metadata across a cluster and produce a report showing + any differences. + + Args: + cluster_name (str): Name of the cluster where the operation should be + performed. + dbname (str, optional): Name of the database. Defaults to the name of + the first database in the cluster configuration. + nodes (str, optional): Comma-delimited subset of nodes on which the + command will be executed. Defaults to "all". + quiet (bool, optional): Whether to suppress output in stdout. Defaults + to False. + + Raises: + AceException: If there's an error specific to the ACE operation. + Exception: For any unexpected errors during the spock diff operation. + + Returns: + None. The function performs the spock diff operation and handles any + exceptions. All output messages are printed to stdout since it's a CLI + function. + """ + task_id = ace_db.generate_task_id() + + try: + sd_task = SpockDiffTask( + cluster_name=cluster_name, + _dbname=dbname, + _nodes=nodes, + quiet_mode=quiet, + invoke_method="cli", + ) + sd_task.scheduler.task_id = task_id + sd_task.scheduler.task_type = "spock-diff" + sd_task.scheduler.task_status = "RUNNING" + sd_task.scheduler.started_at = datetime.now() + + ace.validate_spock_diff_inputs(sd_task) + ace_db.create_ace_task(task=sd_task) + ace_core.spock_diff(sd_task) + sd_task.connection_pool.close_all() + except AceException as e: + util.exit_message(str(e)) + except Exception as e: + traceback.print_exc() + util.exit_message(f"Unexpected error while running spock diff: {e}") + + +class SpockExceptionUpdateCLI(object): + + def __init__(self): + pass + + def run(self, cluster_name, node_name, entry, dbname=None) -> None: + """ + Update the Spock exception status for a specified cluster and node. + + Args: + cluster_name (str): Name of the cluster where the operation should + be performed. + node_name (str): The name of the node within the cluster where the + update should be performed. + entry (str): A JSON string representing the exception entry. Should contain + the following keys. + + - "remote_origin" (str): Identifier of the origin node of the + transaction that caused the exception. (Required) + + - "remote_commit_ts" (str): Commit timestamp of the + transaction on the remote origin. (Required) + + - "remote_xid" (str): Transaction ID on the remote origin. + (Required) + + - "status" (str): The new status to set for the exception (e.g., + "RESOLVED", "IGNORED"). (Required) + + - "resolution_details" (dict, optional): A JSON serialisable + dictionary containing details about the resolution. + + - "command_counter" (int, optional): If specified, only the specific + exception detail (matching this command_counter along with + remote_origin, remote_commit_ts, remote_xid) in the + `spock.exception_status_detail` table is updated. If omitted, + the main entry in `spock.exception_status` and all related + detail entries for the (remote_origin, remote_commit_ts, + remote_xid) trio in `spock.exception_status_detail` are updated. + dbname (str, optional): Name of the database. Defaults to the name of + the first database in the cluster configuration. + + Raises: + AceException: If an error specific to the ACE system occurs. + json.JSONDecodeError: If the provided exception entry is not valid + JSON. + Exception: For any other unexpected errors. + + Returns: + None + """ + try: + conn = ace.update_spock_exception_checks( + cluster_name, node_name, entry, dbname + ) + ace_core.update_spock_exception(entry, conn) + except AceException as e: + util.exit_message(str(e)) + except json.JSONDecodeError: + util.exit_message("Exception entry is not a valid JSON") + except Exception as e: + traceback.print_exc() + util.exit_message(f"Unexpected error while running exception status: {e}") + + util.message("Spock exception status updated successfully", p_state="success") + + +class StartCLI(object): + def __init__(self): + pass + + def run(self): + """ + Start the ACE background scheduler and API. + """ + ace_daemon.start_ace() + + +class AceCLI(object): """ - Start the ACE background scheduler and API. + The Active Consistency Engine of pgEdge. """ - ace_daemon.start_ace() + + def __init__(self): + """ + Initialises the AceCLI and sets up command groups. + """ + self._commands = { + "mtree": MerkleTreeCLI, + "table-diff": TableDiffCLI().run, + "table-repair": TableRepairCLI().run, + "table-rerun": TableRerunCLI().run, + "repset-diff": RepsetDiffCLI().run, + "schema-diff": SchemaDiffCLI().run, + "spock-diff": SpockDiffCLI().run, + "spock-exception-update": SpockExceptionUpdateCLI().run, + "start": StartCLI().run, + } + + def __getattr__(self, name): + try: + # The tests have a convoluted way of invoking the CLI currently, so + # if there is an underscore in the name, we replace it with a dash. + if "_" in name: + name = name.replace("_", "-") + return self._commands[name] + except KeyError: + raise AttributeError(f"No such command: {name}") + + def __dir__(self): + # Allows Fire to generate proper helptext using dash-case commands + return list(self._commands.keys()) diff --git a/cli/scripts/ace_config.py b/cli/scripts/ace_config.py index a7013d85..1580b1c4 100644 --- a/cli/scripts/ace_config.py +++ b/cli/scripts/ace_config.py @@ -9,29 +9,26 @@ # ============================================================================== # Postgres options -STATEMENT_TIMEOUT = 60000 # in milliseconds +STATEMENT_TIMEOUT = 0 # in milliseconds CONNECTION_TIMEOUT = 10 # in seconds - # Default values for ACE table-diff -MAX_DIFF_ROWS = 10000 -MIN_ALLOWED_BLOCK_SIZE = 1000 -MAX_ALLOWED_BLOCK_SIZE = 100000 -BLOCK_ROWS_DEFAULT = os.environ.get("ACE_BLOCK_ROWS", 10000) -MAX_CPU_RATIO_DEFAULT = os.environ.get("ACE_MAX_CPU_RATIO", 0.6) -BATCH_SIZE_DEFAULT = os.environ.get("ACE_BATCH_SIZE", 1) -MAX_BATCH_SIZE = 1000 - - -# Return codes for compare_checksums -BLOCK_OK = 0 -MAX_DIFFS_EXCEEDED = 1 -BLOCK_MISMATCH = 2 -BLOCK_ERROR = 3 +MAX_DIFF_ROWS = 1_000_000 +MIN_DIFF_BLOCK_SIZE = 1000 +MAX_DIFF_BLOCK_SIZE = 100_000 +DIFF_BLOCK_SIZE = os.environ.get("ACE_DIFF_BLOCK_SIZE", 10_000) +MAX_CPU_RATIO = os.environ.get("ACE_MAX_CPU_RATIO", 0.6) +DIFF_BATCH_SIZE = os.environ.get("ACE_BATCH_SIZE", 1) +MAX_DIFF_BATCH_SIZE = 1000 # The minimum version of Spock that supports the repair mode SPOCK_REPAIR_MODE_MIN_VERSION = 4.0 +# ACE Merkle Tree options +MTREE_BLOCK_SIZE = os.environ.get("ACE_MTREE_BLOCK_SIZE", 100_000) +MIN_MTREE_BLOCK_SIZE = 1000 +MAX_MTREE_BLOCK_SIZE = 1_000_000 + # ============================================================================== """ @@ -65,7 +62,7 @@ float between 0 and 1. - batch_size: The batch size to use for the job. How many blocks does a single job process at a time. -- block_rows: The maximum number of rows per block. How many rows does a +- block_size: The maximum number of rows per block. How many rows does a single block contain. Each multiprocessing worker running in parallel will process this many rows at a time. - nodes: A list of node OIDs--if you'd like to run the job only on specific nodes. @@ -94,7 +91,7 @@ "args": { "max_cpu_ratio": 0.7, "batch_size": 1000, - "block_rows": 10000, + "block_size": 10000, "nodes": "all", "output": "json", "quiet": False, @@ -108,7 +105,7 @@ "args": { "max_cpu_ratio": 0.7, "batch_size": 1000, - "block_rows": 10000, + "block_size": 10000, "nodes": "all", "output": "json", "quiet": False, diff --git a/cli/scripts/ace_constants.py b/cli/scripts/ace_constants.py new file mode 100644 index 00000000..aa6729eb --- /dev/null +++ b/cli/scripts/ace_constants.py @@ -0,0 +1,5 @@ +# Return codes for diff workers +BLOCK_OK = 0 +MAX_DIFFS_EXCEEDED = 1 +BLOCK_MISMATCH = 2 +BLOCK_ERROR = 3 diff --git a/cli/scripts/ace_core.py b/cli/scripts/ace_core.py index 8983ea08..91567668 100644 --- a/cli/scripts/ace_core.py +++ b/cli/scripts/ace_core.py @@ -1,3 +1,4 @@ +import itertools import json from math import ceil import os @@ -10,8 +11,8 @@ from multiprocessing import Manager, cpu_count from mpire import WorkerPool from mpire.utils import make_single_arguments -from ordered_set import OrderedSet from psycopg import sql +from psycopg import ClientCursor from psycopg.rows import dict_row, class_row from dateutil import parser @@ -19,7 +20,6 @@ import ace import ace_db import ace_html_reporter -import pgpasslib import cluster import util import ace_config as config @@ -34,89 +34,208 @@ ) from ace_exceptions import AceException, AuthenticationError +from ace_constants import BLOCK_OK, BLOCK_MISMATCH, BLOCK_ERROR -def run_query(worker_state, host, query): - cur = worker_state[host] +def run_query(worker_state, host, query, stmt_name=None, params=None): + if stmt_name: + cur = worker_state["cursors"][host] + + prepared_sql = sql.SQL("EXECUTE {stmt_name} ({params})").format( + stmt_name=sql.Identifier(stmt_name), + params=sql.SQL(", ").join(sql.Literal(param) for param in params), + ) + cur.execute(prepared_sql) + results = cur.fetchall() + return results + + cur = worker_state["cursors"][host] cur.execute(query) results = cur.fetchall() return results -# FIXME: Replace with td_task.connection_pool.connect() after merkle trees -# PR is merged -def init_conn_pool(shared_objects, worker_state): +def init_conn_pool(worker_id, shared_objects, worker_state): + task = shared_objects["task"] - td_task = shared_objects["td_task"] + worker_state["cursors"] = {} + worker_state["prepared_statements"] = {} - for node in td_task.fields.cluster_nodes: + p_key = task.fields.key + schema_name = task.fields.l_schema + table_name = task.fields.l_table + cols = task.fields.cols.split(",") + simple_primary_key = task.fields.simple_primary_key + + for node in task.fields.cluster_nodes: try: - host = node["public_ip"] - port = node.get("port", 5432) - db_user = node["db_user"] - dbname = node["db_name"] - - params = { - "dbname": dbname, - "user": db_user, - "host": host, - "port": port, - "options": f"-c statement_timeout={config.STATEMENT_TIMEOUT}", - "application_name": "ACE", - "keepalives": 1, - "keepalives_idle": 30, - "keepalives_interval": 10, - "keepalives_count": 5, - "connect_timeout": config.CONNECTION_TIMEOUT, - } + _, conn = task.connection_pool.connect(node) + task.connection_pool.drop_privileges( + conn, + client_role=( + task.client_role + if config.USE_CERT_AUTH or task.invoke_method == "api" + else None + ), + ) + cur = conn.cursor() + worker_state["cursors"][node["name"]] = cur - if config.USE_CERT_AUTH: - if config.ACE_USER_CERT_FILE and config.ACE_USER_KEY_FILE: - params.update( - { - "sslmode": "verify-full", - "sslcert": config.ACE_USER_CERT_FILE, - "sslkey": config.ACE_USER_KEY_FILE, - "sslrootcert": config.CA_CERT_FILE, - } + worker_state["prepared_statements"][node["name"]] = {} + host = node["name"] + + if simple_primary_key: + cur.execute( + sql.SQL( + """ + PREPARE hash_simple AS + WITH block_rows AS ( + SELECT * + FROM {schema}.{table} + WHERE + ($1::boolean OR {p_key} >= $3) AND + ($2::boolean OR {p_key} < $4) + ), + block_hash AS ( + SELECT + digest( + COALESCE( + string_agg( + concat_ws('|', {columns}), + '|' + ORDER BY {key} + ), + 'EMPTY_BLOCK' + ), + 'sha256' + ) as leaf_hash + FROM block_rows + ) + SELECT encode(leaf_hash, 'hex') as leaf_hash + FROM block_hash + """ + ).format( + schema=sql.Identifier(schema_name), + table=sql.Identifier(table_name), + p_key=sql.Identifier(p_key), + key=sql.Identifier(p_key), + columns=sql.SQL(", ").join(sql.Identifier(col) for col in cols), ) - else: - raise AuthenticationError( - "Client certificate authentication is enabled but no" - "certificate files are provided" + ) + worker_state["prepared_statements"][host]["hash_simple"] = "hash_simple" + + cur.execute( + sql.SQL( + """ + PREPARE block_simple AS + SELECT * FROM {schema}.{table} + WHERE + ($1::boolean OR {p_key} >= $3) AND + ($2::boolean OR {p_key} < $4) + """ + ).format( + schema=sql.Identifier(schema_name), + table=sql.Identifier(table_name), + p_key=sql.Identifier(p_key), ) + ) + worker_state["prepared_statements"][host][ + "block_simple" + ] = "block_simple" + else: - if td_task.client_role is not None: - raise AuthenticationError( - "Client certificate authentication needs to be enabled" - "for API usage" + p_key_cols = [col.strip() for col in p_key.split(",")] + p_key_count = len(p_key_cols) + + pk_cols_sql = sql.SQL(", ").join( + sql.Identifier(col) for col in p_key_cols + ) + + # $1 and $2 are for skip_min_check and skip_max_check, + # $3, $4, ... are for min pkey values + placeholders1 = sql.SQL(", ").join( + sql.SQL("${i}").format(i=i) for i in range(3, p_key_count + 3) + ) + + placeholders2 = sql.SQL(", ").join( + sql.SQL("${i}").format(i=i) + for i in range(p_key_count + 3, 2 * p_key_count + 3) + ) + + cur.execute( + sql.SQL( + """ + PREPARE hash_composite_full AS + WITH block_rows AS ( + SELECT * + FROM {schema}.{table} + WHERE + ($1::boolean OR ({pk_cols}) >= ({placeholders1})) AND + ($2::boolean OR + ({pk_cols}) < ({placeholders2})) + ), + block_hash AS ( + SELECT + digest( + COALESCE( + string_agg( + concat_ws('|', {columns}), + '|' + ORDER BY {order_by} + ), + 'EMPTY_BLOCK' + ), + 'sha256' + ) as leaf_hash + FROM block_rows + ) + SELECT encode(leaf_hash, 'hex') as leaf_hash + FROM block_hash + """ + ).format( + schema=sql.Identifier(schema_name), + table=sql.Identifier(table_name), + pk_cols=pk_cols_sql, + placeholders1=placeholders1, + placeholders2=placeholders2, + columns=sql.SQL(", ").join(sql.Identifier(col) for col in cols), + order_by=pk_cols_sql, ) - # Handle password authentication - if node["db_password"]: - params["password"] = node["db_password"] - else: - pgpass = pgpasslib.getpass( - host=node["name"], - user=db_user, - dbname=dbname, - port=port, + ) + worker_state["prepared_statements"][host][ + "hash_composite_full" + ] = "hash_composite_full" + + cur.execute( + sql.SQL( + """ + PREPARE block_composite_full AS + SELECT * FROM {schema}.{table} + WHERE + ($1::boolean OR ({pk_cols}) >= ({placeholders1})) AND + ($2::boolean OR ({pk_cols}) < ({placeholders2})) + """ + ).format( + schema=sql.Identifier(schema_name), + table=sql.Identifier(table_name), + pk_cols=pk_cols_sql, + placeholders1=placeholders1, + placeholders2=placeholders2, ) - if not pgpass: - msg = f"No password found for {node['name']}" - raise AuthenticationError(msg) - params["password"] = pgpass + ) + worker_state["prepared_statements"][host][ + "block_composite_full" + ] = "block_composite_full" - conn = psycopg.connect(**params) - worker_state[node["name"]] = conn.cursor() except AuthenticationError as e: raise AceException(str(e)) # Ignore the type checker warning for shared_objects. # It is unused in this function, but is required by the mpire library. -def close_conn_pool(shared_objects, worker_state): +def close_conn_pool(worker_id, shared_objects, worker_state): try: - for host, cur in worker_state.items(): + for _, cur in worker_state.get("cursors", {}).items(): conn = cur.connection cur.close() conn.close() @@ -127,17 +246,18 @@ def close_conn_pool(shared_objects, worker_state): # Accepts list of pkeys and values and generates a where clause that in the form # `(pkey1name, pkey2name ...) in ( (pkey1val1, pkey2val1 ...), # (pkey1val2, pkey2val2 ...) ... )` -def generate_where_clause(primary_keys, id_values): +def generate_where_clause(primary_keys, batches): if len(primary_keys) == 1: # Single primary key - id_values_list = ", ".join(repr(val) for val in id_values) + id_values_list = ", ".join(repr(val) for val in itertools.chain(*batches)) # Wrap column name in double quotes to preserve case query = f'"{primary_keys[0]}" IN ({id_values_list})' else: # Composite primary key conditions = ", ".join( - f"({', '.join(repr(val) for val in id_tuple)})" for id_tuple in id_values + f"({', '.join(repr(val) for val in id_tuple)})" + for id_tuple in itertools.chain(*batches) ) # Wrap each column name in double quotes to preserve case key_columns = ", ".join(f'"{key}"' for key in primary_keys) @@ -164,113 +284,137 @@ def create_result_dict( } -def compare_checksums(shared_objects, worker_state, batches): +def compare_checksums(worker_id, shared_objects, worker_state, pkey1, pkey2): + """ + Same approach as compare_ranges in ace_mtree.py + Use lookup dictionaries for faster comparisons, and process all batches + at once if possible + """ + + task = shared_objects["task"] + mode = shared_objects["mode"] + stop_event = shared_objects["stop_event"] - result_queue = shared_objects["result_queue"] - diff_dict = shared_objects["diff_dict"] - row_diff_count = shared_objects["row_diff_count"] - lock = shared_objects["lock"] + p_key = task.fields.key + schema_name = task.fields.l_schema + table_name = task.fields.l_table + node_list = task.fields.node_list + cols = task.fields.cols.split(",") + simple_primary_key = task.fields.simple_primary_key - if row_diff_count.value >= config.MAX_DIFF_ROWS: + worker_diffs = {} + total_diffs = 0 + + if stop_event.is_set(): return - p_key = shared_objects["p_key"] - schema_name = shared_objects["schema_name"] - table_name = shared_objects["table_name"] - node_list = shared_objects["node_list"] - cols = shared_objects["cols_list"] - simple_primary_key = shared_objects["simple_primary_key"] - mode = shared_objects["mode"] + # We can use prepared statements for diff mode with a single batch + use_prepared = mode == "diff" + params = [] - for batch in batches: - where_clause = str() - where_clause_temp = list() + if mode == "diff": - if mode == "diff": - pkey1, pkey2 = batch - if simple_primary_key: - if pkey1 is not None: - where_clause_temp.append( - sql.SQL("{p_key} >= {pkey1}").format( - p_key=sql.Identifier(p_key), pkey1=sql.Literal(pkey1) - ) - ) - if pkey2 is not None: - where_clause_temp.append( - sql.SQL("{p_key} < {pkey2}").format( - p_key=sql.Identifier(p_key), pkey2=sql.Literal(pkey2) - ) - ) - else: - """ - This is a slightly more complicated case since we have to split up - the primary key and compare them with split values of pkey1 and pkey2 - """ + if simple_primary_key: + has_min = pkey1 and pkey1[0] is not None + min_key = pkey1[0] if has_min else None + + has_max = pkey2 and pkey2[0] is not None + max_key = pkey2[0] if has_max else None + + # [skip_min_check, min_value, skip_max_check, max_value] + # skip_min_check and skip_max_check work as follows: + # $1::boolean OR {p_key} >= + # $2::boolean OR {p_key} < + # + # For our first range, start is None, so we don't need to check + # for min. For the last range, end is None, so we don't need + # to check for max. + params = [not has_min, not has_max, min_key, max_key] - if pkey1 is not None: - where_clause_temp.append( - sql.SQL("({p_key}) >= ({pkey1})").format( - p_key=sql.SQL(", ").join( - [ - sql.Identifier(col.strip()) - for col in p_key.split(",") - ] - ), - pkey1=sql.SQL(", ").join( - [sql.Literal(val) for val in pkey1] - ), - ) - ) + else: + p_key_cols = [col.strip() for col in p_key.split(",")] + p_key_count = len(p_key_cols) - if pkey2 is not None: - where_clause_temp.append( - sql.SQL("({p_key}) < ({pkey2})").format( - p_key=sql.SQL(", ").join( - [ - sql.Identifier(col.strip()) - for col in p_key.split(",") - ] - ), - pkey2=sql.SQL(", ").join( - [sql.Literal(val) for val in pkey2] - ), - ) - ) + has_min = pkey1 and pkey1[0] is not None + min_values = list(pkey1[0]) if has_min else [None] * p_key_count - where_clause = sql.SQL(" AND ").join(where_clause_temp) + has_max = pkey2 and pkey2[0] is not None + max_values = list(pkey2[0]) if has_max else [None] * p_key_count - elif mode == "rerun": - keys = p_key.split(",") - where_clause = sql.SQL(generate_where_clause(keys, batch)) + params = [not has_min, not has_max] + min_values + max_values - else: - raise Exception(f"Mode {mode} not recognized in compare_checksums") + elif mode == "rerun": + # Again, falling back to dynamically generated queries for rerun mode + keys = p_key.split(",") + where_clause = sql.SQL(generate_where_clause(keys, [pkey1, pkey2])) if simple_primary_key: hash_sql = sql.SQL( - "SELECT md5(cast(array_agg(t.* ORDER BY {p_key}) AS text)) FROM" - "(SELECT * FROM {table_name} WHERE {where_clause}) t" - ).format( - p_key=sql.Identifier(p_key), - table_name=sql.SQL("{}.{}").format( - sql.Identifier(schema_name), - sql.Identifier(table_name), + """ + WITH block_rows AS ( + SELECT * + FROM {schema}.{table} + WHERE {where_clause} ), + block_hash AS ( + SELECT + digest( + COALESCE( + string_agg( + concat_ws('|', {columns}), + '|' + ORDER BY {key} + ), + 'EMPTY_BLOCK' + ), + 'sha256' + ) as leaf_hash + FROM block_rows + ) + SELECT encode(leaf_hash, 'hex') as leaf_hash + FROM block_hash + """ + ).format( + schema=sql.Identifier(schema_name), + table=sql.Identifier(table_name), where_clause=where_clause, + key=sql.Identifier(p_key), + columns=sql.SQL(", ").join(sql.Identifier(col) for col in cols), ) else: hash_sql = sql.SQL( - "SELECT md5(cast(array_agg(t.* ORDER BY {p_key}) AS text)) FROM" - "(SELECT * FROM {table_name} WHERE {where_clause}) t" - ).format( - p_key=sql.SQL(", ").join( - [sql.Identifier(col.strip()) for col in p_key.split(",")] - ), - table_name=sql.SQL("{}.{}").format( - sql.Identifier(schema_name), - sql.Identifier(table_name), + """ + WITH block_rows AS ( + SELECT * + FROM {schema}.{table} + WHERE {where_clause} ), + block_hash AS ( + SELECT + digest( + COALESCE( + string_agg( + concat_ws('|', {columns}), + '|' + ORDER BY {key} + ), + 'EMPTY_BLOCK' + ), + 'sha256' + ) as leaf_hash + FROM block_rows + ) + SELECT encode(leaf_hash, 'hex') as leaf_hash + FROM block_hash + """ + ).format( + schema=sql.Identifier(schema_name), + table=sql.Identifier(table_name), where_clause=where_clause, + key=sql.SQL(", ").join( + sql.Identifier(col.strip()) for col in p_key.split(",") + ), + columns=sql.SQL(", ").join(sql.Identifier(col) for col in cols), ) block_sql = sql.SQL("SELECT * FROM {table_name} WHERE {where_clause}").format( @@ -281,156 +425,171 @@ def compare_checksums(shared_objects, worker_state, batches): where_clause=where_clause, ) - for node_pair in combinations(node_list, 2): - host1 = node_pair[0] - host2 = node_pair[1] - - # Return early if we have already exceeded the max number of diffs - if row_diff_count.value >= config.MAX_DIFF_ROWS: - result_dict = create_result_dict( - node_pair, - batch, - config.MAX_DIFFS_EXCEEDED, - "MAX_DIFFS_EXCEEDED", - errors=True, - error_messages=[ - f"Diffs have exceeded the maximum allowed number of diffs:" - f"{config.MAX_DIFF_ROWS}" - ], - ) - result_queue.append(result_dict) - return config.MAX_DIFFS_EXCEEDED + else: + raise Exception(f"Mode {mode} not recognized in compare_checksums") - # Run the checksum query on both nodes in parallel - with ThreadPoolExecutor(max_workers=2) as executor: + for node_pair in combinations(node_list, 2): + host1 = node_pair[0] + host2 = node_pair[1] + node_pair_key = f"{host1}/{host2}" + + with ThreadPoolExecutor(max_workers=2) as executor: + if use_prepared: + stmt_name = ( + "hash_simple" if simple_primary_key else "hash_composite_full" + ) + futures = [ + executor.submit( + run_query, + worker_state, + host1, + None, + stmt_name=stmt_name, + params=params, + ), + executor.submit( + run_query, + worker_state, + host2, + None, + stmt_name=stmt_name, + params=params, + ), + ] + else: futures = [ executor.submit(run_query, worker_state, host1, hash_sql), executor.submit(run_query, worker_state, host2, hash_sql), ] - results = [f.result() for f in futures if not f.exception()] - errors = [f.exception() for f in futures if f.exception()] + results = [f.result() for f in futures if not f.exception()] - if errors: - result_dict = create_result_dict( - node_pair, - batch, - config.BLOCK_ERROR, - "BLOCK_ERROR", - errors=True, - error_messages=[str(error) for error in errors], - ) - result_queue.append(result_dict) - return config.BLOCK_ERROR + errors = [f.exception() for f in futures if f.exception()] - hash1, hash2 = results[0][0][0], results[1][0][0] + if errors: + return { + "status": BLOCK_ERROR, + "errors": [str(error) for error in errors], + "node_pair": node_pair, + } + + if len(results) < 2: + return { + "status": BLOCK_ERROR, + "errors": ["Failed to get results from both nodes"], + "node_pair": node_pair, + } - if hash1 != hash2: - # Run the block query on both nodes in parallel - with ThreadPoolExecutor(max_workers=2) as executor: + hash1, hash2 = results[0][0][0], results[1][0][0] + + if hash1 != hash2: + with ThreadPoolExecutor(max_workers=2) as executor: + if use_prepared: + stmt_name = ( + "block_simple" if simple_primary_key else "block_composite_full" + ) + futures = [ + executor.submit( + run_query, + worker_state, + host1, + None, + stmt_name=stmt_name, + params=params, + ), + executor.submit( + run_query, + worker_state, + host2, + None, + stmt_name=stmt_name, + params=params, + ), + ] + else: futures = [ executor.submit(run_query, worker_state, host1, block_sql), executor.submit(run_query, worker_state, host2, block_sql), ] - results = [f.result() for f in futures if not f.exception()] - - errors = [f.exception() for f in futures if f.exception()] - - if errors: - result_dict = create_result_dict( - node_pair, - batch, - config.BLOCK_ERROR, - "BLOCK_ERROR", - errors=True, - error_messages=[str(error) for error in errors], - ) - result_queue.append(result_dict) - return config.BLOCK_ERROR - t1_result, t2_result = results - - # Transform all elements in t1_result and t2_result into strings before - # consolidating them into a set - # TODO: Test and add support for different datatypes here - t1_result = [ - tuple(x.hex() if isinstance(x, bytes) else str(x) for x in row) - for row in t1_result - ] - t2_result = [ - tuple(x.hex() if isinstance(x, bytes) else str(x) for x in row) - for row in t2_result - ] + results = [f.result() for f in futures if not f.exception()] - # Collect results into OrderedSets for comparison - t1_set = OrderedSet(t1_result) - t2_set = OrderedSet(t2_result) + errors = [f.exception() for f in futures if f.exception()] - t1_diff = t1_set - t2_set - t2_diff = t2_set - t1_set + if errors: + return { + "status": BLOCK_ERROR, + "errors": [str(error) for error in errors], + "node_pair": node_pair, + } - # It is possible that the hash mismatch is a false negative. - # E.g., if there are extraneous spaces in the JSONB column. - # In this case, we can still consider the block to be OK. - if (not t1_diff and not t2_diff) or ( - len(t1_diff) == 0 and len(t2_diff) == 0 - ): - result_dict = create_result_dict( - node_pair, batch, config.BLOCK_OK, "BLOCK_OK" - ) - result_queue.append(result_dict) - continue + if len(results) < 2: + return { + "status": BLOCK_ERROR, + "errors": ["Failed to get results from both nodes"], + "node_pair": node_pair, + } - node_pair_key = f"{host1}/{host2}" + t1_result, t2_result = results - if node_pair_key not in diff_dict: - diff_dict[node_pair_key] = {} + t1_dict = {} + t2_dict = {} - with lock: - # Update diff_dict with the results of the diff - if len(t1_diff) > 0 or len(t2_diff) > 0: - temp_dict = {} - if host1 in diff_dict[node_pair_key]: - temp_dict[host1] = diff_dict[node_pair_key][host1] - else: - temp_dict[host1] = [] - if host2 in diff_dict[node_pair_key]: - temp_dict[host2] = diff_dict[node_pair_key][host2] - else: - temp_dict[host2] = [] - - temp_dict[host1] += [dict(zip(cols, row)) for row in t1_diff] - temp_dict[host2] += [dict(zip(cols, row)) for row in t2_diff] - - diff_dict[node_pair_key] = temp_dict - - # Update row_diff_count with the number of diffs - row_diff_count.value += max(len(t1_diff), len(t2_diff)) - - if row_diff_count.value >= config.MAX_DIFF_ROWS: - result_dict = create_result_dict( - node_pair, - batch, - config.MAX_DIFFS_EXCEEDED, - "MAX_DIFFS_EXCEEDED", - errors=True, - error_messages=[ - f"Diffs have exceeded the maximum allowed number of diffs:" - f"{config.MAX_DIFF_ROWS}" - ], + for row in t1_result: + row_key = tuple( + ( + x.hex() + if isinstance(x, bytes) + else str(x) if isinstance(x, (dict, list, set)) else x ) - result_queue.append(result_dict) - return config.MAX_DIFFS_EXCEEDED - else: - result_dict = create_result_dict( - node_pair, batch, config.BLOCK_MISMATCH, "BLOCK_MISMATCH" + for x in row + ) + t1_dict[row_key] = row + + t2_only = [] + for row in t2_result: + row_key = tuple( + ( + x.hex() + if isinstance(x, bytes) + else str(x) if isinstance(x, (dict, list, set)) else x ) - result_queue.append(result_dict) - else: - result_dict = create_result_dict( - node_pair, batch, config.BLOCK_OK, "BLOCK_OK" + for x in row + ) + t2_dict[row_key] = row + + if row_key not in t1_dict: + t2_only.append(row_key) + + t1_only = [key for key in t1_dict.keys() if key not in t2_dict] + + # It is possible that the hash mismatch is a false negative. + # E.g., if there are extraneous spaces in the JSONB column. + # In this case, we can still consider the block to be OK. + if not t1_only and not t2_only: + continue + + if node_pair_key not in worker_diffs: + worker_diffs[node_pair_key] = {host1: [], host2: []} + + for row_key in t1_only: + worker_diffs[node_pair_key][host1].append( + dict(zip(cols, (str(x) for x in row_key))) ) - result_queue.append(result_dict) + + for row_key in t2_only: + worker_diffs[node_pair_key][host2].append( + dict(zip(cols, (str(x) for x in row_key))) + ) + + total_diffs += max(len(t1_only), len(t2_only)) + + return { + "status": BLOCK_OK if total_diffs == 0 else BLOCK_MISMATCH, + "diffs": worker_diffs, + "total_diffs": total_diffs, + "node_pair": node_pair_key if "node_pair_key" in locals() else None, + } def table_diff(td_task: TableDiffTask, skip_all_checks: bool = False): @@ -445,14 +604,12 @@ def table_diff(td_task: TableDiffTask, skip_all_checks: bool = False): "mismatch": False, "errors": [ f"Error during pre-flight checks for table_diff: {str(e_checks)}" - ] + ], } ace.handle_task_exception(td_task, context) raise - simple_primary_key = True - if len(td_task.fields.key.split(",")) > 1: - simple_primary_key = False + simple_primary_key = td_task.fields.simple_primary_key row_count = 0 total_rows = 0 @@ -470,18 +627,22 @@ def table_diff(td_task: TableDiffTask, skip_all_checks: bool = False): } _, conn = td_task.connection_pool.get_cluster_node_connection( node_info, - td_task.cluster_name, - invoke_method=td_task.invoke_method, client_role=( td_task.client_role - if (config.USE_CERT_AUTH and td_task.invoke_method == "api") + if (config.USE_CERT_AUTH or td_task.invoke_method == "api") else None ), ) - rows = ace.get_row_count( - conn, td_task.fields.l_schema, td_task.fields.l_table - ) + if td_task.table_filter: + rows = ace.get_row_count( + conn, td_task.fields.l_schema, td_task.fields.l_table + ) + else: + rows = ace.get_row_count_estimate( + conn, td_task.fields.l_schema, td_task.fields.l_table + ) + total_rows += rows if rows > row_count: row_count = rows @@ -503,18 +664,48 @@ def table_diff(td_task: TableDiffTask, skip_all_checks: bool = False): ) return - # Use conn_with_max_rows to get the first and last primary key values - # of every block row. Repeat until we no longer have any more rows. - # Store results in pkey_offsets. + print("Estimated row count:", row_count) - if simple_primary_key: - pkey_sql = sql.SQL("SELECT {key} FROM {table_name} ORDER BY {key}").format( - key=sql.Identifier(td_task.fields.key), - table_name=sql.SQL("{}.{}").format( - sql.Identifier(td_task.fields.l_schema), - sql.Identifier(td_task.fields.l_table), - ), - ) + util.message( + "Getting primary key offsets for table...", + p_state="info", + quiet_mode=td_task.quiet_mode, + ) + + total_blocks = row_count // td_task.block_size + total_blocks = total_blocks if total_blocks > 0 else 1 + cpus = cpu_count() + max_procs = int(cpus * td_task.max_cpu_ratio) if cpus > 1 else 1 + + # If we don't have enough blocks to keep all CPUs busy, use fewer processes + procs = max_procs if total_blocks > max_procs else total_blocks + + if (row_count > 10**4) and (not td_task.table_filter): + sample_method, sample_percent = ace.compute_sampling_parameters(row_count) + + print(f"Using {sample_method} sampling with {sample_percent}% of rows") + + try: + ref_cur = conn_with_max_rows.cursor() + offsets_query = ace.generate_pkey_offsets_query( + schema=td_task.fields.l_schema, + table=td_task.fields.l_table, + key_columns=td_task.fields.key.split(","), + table_sample_method=sample_method, + sample_percent=sample_percent, + ntile_count=total_blocks, + ) + ref_cur.execute(offsets_query) + raw_offsets = ref_cur.fetchall() + pkey_offsets = ace.process_pkey_offsets(raw_offsets) + except Exception as e: + context = { + "total_rows": total_rows, + "mismatch": False, + "errors": [str(e)], + } + ace.handle_task_exception(td_task, context) + raise e else: pkey_sql = sql.SQL("SELECT {key} FROM {table_name} ORDER BY {key}").format( key=sql.SQL(", ").join( @@ -526,75 +717,54 @@ def table_diff(td_task: TableDiffTask, skip_all_checks: bool = False): ), ) - def get_pkey_offsets(conn, pkey_sql, block_rows): - pkey_offsets = [] - cur = conn.cursor() - cur.execute(pkey_sql) - rows = cur.fetchmany(block_rows) - - if simple_primary_key: - rows[:] = [str(x[0]) for x in rows] - pkey_offsets.append((None, str(rows[0]))) - prev_min_offset = str(rows[0]) - prev_max_offset = str(rows[-1]) - else: - rows[:] = [tuple(str(i) for i in x) for x in rows] - pkey_offsets.append((None, rows[0])) - prev_min_offset = rows[0] - prev_max_offset = rows[-1] - - while rows: + def get_pkey_offsets(conn, pkey_sql, block_rows): + pkey_offsets = [] + cur = conn.cursor() + cur.execute(pkey_sql) rows = cur.fetchmany(block_rows) + if simple_primary_key: rows[:] = [str(x[0]) for x in rows] + pkey_offsets.append(([None], [str(rows[0])])) + prev_min_offset = str(rows[0]) + prev_max_offset = str(rows[-1]) else: rows[:] = [tuple(str(i) for i in x) for x in rows] + pkey_offsets.append(([None], [rows[0]])) + prev_min_offset = rows[0] + prev_max_offset = rows[-1] - if not rows: - if prev_max_offset != prev_min_offset: - pkey_offsets.append((prev_min_offset, prev_max_offset)) - pkey_offsets.append((prev_max_offset, None)) - break + while rows: + rows = cur.fetchmany(block_rows) + if simple_primary_key: + rows[:] = [str(x[0]) for x in rows] + else: + rows[:] = [tuple(str(i) for i in x) for x in rows] - curr_min_offset = rows[0] - pkey_offsets.append((prev_min_offset, curr_min_offset)) - prev_min_offset = curr_min_offset - prev_max_offset = rows[-1] + if not rows: + if prev_max_offset != prev_min_offset: + pkey_offsets.append(([prev_min_offset], [prev_max_offset])) + pkey_offsets.append(([prev_max_offset], [None])) + break - cur.close() - conn.close() - return pkey_offsets + curr_min_offset = rows[0] + pkey_offsets.append(([prev_min_offset], [curr_min_offset])) + prev_min_offset = curr_min_offset + prev_max_offset = rows[-1] - util.message( - "Getting primary key offsets for table...", - p_state="info", - quiet_mode=td_task.quiet_mode, - ) + cur.close() + conn.close() + return pkey_offsets - future = ThreadPoolExecutor().submit( - get_pkey_offsets, conn_with_max_rows, pkey_sql, td_task.block_rows - ) - pkey_offsets = future.result() if not future.exception() else [] - if future.exception(): - context = { - "total_rows": total_rows, - "mismatch": False, - "errors": [str(future.exception())], - } - ace.handle_task_exception(td_task, context) - raise future.exception() + pkey_offsets = get_pkey_offsets( + conn_with_max_rows, pkey_sql, td_task.block_size + ) # We're done with getting table metadata. Closing all connections. for conn in conn_list: conn.close() - total_blocks = row_count // td_task.block_rows - total_blocks = total_blocks if total_blocks > 0 else 1 - cpus = cpu_count() - max_procs = int(cpus * td_task.max_cpu_ratio) if cpus > 1 else 1 - - # If we don't have enough blocks to keep all CPUs busy, use fewer processes - procs = max_procs if total_blocks > max_procs else total_blocks + td_task.connection_pool.close_all() start_time = datetime.now() @@ -604,33 +774,16 @@ def get_pkey_offsets(conn, pkey_sql, block_rows): to capture diffs even if rows are absent in one node """ - cols_list = td_task.fields.cols.split(",") - cols_list = [col for col in cols_list if not col.startswith("_Spock_")] + diff_dict = {} - # Shared multiprocessing data structures - result_queue = Manager().list() - diff_dict = Manager().dict() - row_diff_count = Manager().Value("I", 0) - lock = Manager().Lock() + stop_event = Manager().Event() # TODO: Clean this up # Shared variables needed by all workers shared_objects = { - "cluster_name": td_task.cluster_name, - "database": td_task.fields.database, - "node_list": td_task.fields.node_list, - "schema_name": td_task.fields.l_schema, - "table_name": td_task.fields.l_table, - "cols_list": cols_list, - "p_key": td_task.fields.key, - "block_rows": td_task.block_rows, - "simple_primary_key": simple_primary_key, "mode": "diff", - "result_queue": result_queue, - "diff_dict": diff_dict, - "row_diff_count": row_diff_count, - "lock": lock, - "td_task": td_task, + "task": td_task, + "stop_event": stop_event, } util.message( @@ -639,39 +792,55 @@ def get_pkey_offsets(conn, pkey_sql, block_rows): quiet_mode=td_task.quiet_mode, ) - batches = [ - pkey_offsets[i : i + td_task.batch_size] - for i in range(0, len(pkey_offsets), td_task.batch_size) - ] - mismatch = False diffs_exceeded = False errors = False error_list = [] + total_diffs = 0 try: with WorkerPool( n_jobs=procs, shared_objects=shared_objects, use_worker_state=True, + pass_worker_id=True, ) as pool: for result in pool.imap_unordered( compare_checksums, - make_single_arguments(batches), + pkey_offsets, worker_init=init_conn_pool, worker_exit=close_conn_pool, progress_bar=True if not td_task.quiet_mode else False, - iterable_len=len(batches), progress_bar_style="rich", + iterable_len=len(pkey_offsets), ): - if result == config.MAX_DIFFS_EXCEEDED: - diffs_exceeded = True - mismatch = True - break - elif result == config.BLOCK_ERROR: + if result["status"] == BLOCK_ERROR: errors = True + error_list.append( + { + "node_pair": result["node_pair"], + "errors": result["errors"], + } + ) + stop_event.set() break + if result["status"] == BLOCK_MISMATCH: + mismatch = True + for node_pair, node_diffs in result["diffs"].items(): + if node_pair not in diff_dict: + diff_dict[node_pair] = {} + for node, diffs in node_diffs.items(): + if node not in diff_dict[node_pair]: + diff_dict[node_pair][node] = [] + diff_dict[node_pair][node].extend(diffs) + + total_diffs += result["total_diffs"] + if total_diffs >= config.MAX_DIFF_ROWS: + diffs_exceeded = True + stop_event.set() + break + if diffs_exceeded: util.message( "Prematurely terminated jobs since diffs have" @@ -697,13 +866,6 @@ def get_pkey_offsets(conn, pkey_sql, block_rows): "errors": [], } - for result in result_queue: - if result["status_code"] == config.BLOCK_MISMATCH: - mismatch = True - elif result["status_code"] == config.BLOCK_ERROR: - errors = True - error_list.append(result) - if errors: context = {"total_rows": total_rows, "mismatch": mismatch, "errors": error_list} ace.handle_task_exception(td_task, context) @@ -797,11 +959,9 @@ def get_pkey_offsets(conn, pkey_sql, block_rows): } _, conn = td_task.connection_pool.get_cluster_node_connection( node_info, - td_task.cluster_name, - invoke_method=td_task.invoke_method, client_role=( td_task.client_role - if config.USE_CERT_AUTH and td_task.invoke_method == "api" + if (config.USE_CERT_AUTH or td_task.invoke_method == "api") else None ), ) @@ -857,11 +1017,9 @@ def table_repair(tr_task: TableRepairTask): # spock.repair_mode(true) and session_replication_role _, conn = tr_task.connection_pool.get_cluster_node_connection( node_info, - tr_task.cluster_name, - invoke_method=tr_task.invoke_method, client_role=( tr_task.client_role - if config.USE_CERT_AUTH and tr_task.invoke_method == "api" + if (config.USE_CERT_AUTH or tr_task.invoke_method == "api") else None ), drop_privileges=False, @@ -1123,6 +1281,9 @@ def generate_report(): for divergent_node in other_nodes: + total_upserted[divergent_node] = 0 + total_deleted[divergent_node] = 0 + try: conn = conns[divergent_node] cur = conn.cursor() @@ -1157,7 +1318,7 @@ def generate_report(): conn, ( tr_task.client_role - if config.USE_CERT_AUTH and tr_task.invoke_method == "api" + if (config.USE_CERT_AUTH or tr_task.invoke_method == "api") else None ), ) @@ -1220,18 +1381,111 @@ def generate_report(): ) try: - # Let's first delete rows to avoid secondary unique key violations + client_cur = ClientCursor(cur.connection) if delete_keys and not tr_task.upsert_only and not tr_task.insert_only: - # Performing the deletes - cur.executemany(delete_sql, delete_keys) + table_ident = sql.Identifier( + tr_task.fields.l_schema, tr_task.fields.l_table + ) + + if simple_primary_key: + pkey_ident = sql.Identifier(keys_list[0]) + values_sql = sql.SQL(", ").join( + sql.Literal(k[0]) for k in delete_keys + ) + where_clause = sql.SQL("{pkey} IN ({values})").format( + pkey=pkey_ident, values=values_sql + ) + else: + pkey_idents = sql.SQL(", ").join( + sql.Identifier(k) for k in keys_list + ) + values_sql = sql.SQL("({})").format( + sql.SQL(", ").join(sql.Placeholder() * len(keys_list)) + ) + # pdb.set_trace() + mogrified_values = ", ".join( + client_cur.mogrify(values_sql, row) for row in delete_keys + ) + + where_clause = sql.SQL("({pkeys}) IN ({values})").format( + pkeys=pkey_idents, values=sql.SQL(mogrified_values) + ) + + delete_sql = sql.SQL("DELETE FROM {table} WHERE {where};").format( + table=table_ident, where=where_clause + ) + # nosemgrep + cur.execute(delete_sql) + total_deleted[divergent_node] = cur.rowcount if cur.rowcount else 0 + if tr_task.generate_report: + if divergent_node not in report["changes"]: + report["changes"][divergent_node] = {} report["changes"][divergent_node]["deleted_rows"] = delete_keys + elif delete_keys and (tr_task.upsert_only or tr_task.insert_only): deletes_skipped[divergent_node] = delete_keys - # Now perform the upsert - cur.executemany(update_sql, upsert_tuples) + upsert_tuples = ace.convert_json_to_pg_type( + rows_to_upsert_json, cols_list, col_types + ) + + if upsert_tuples: + table_ident = sql.Identifier( + tr_task.fields.l_schema, tr_task.fields.l_table + ) + cols_ident = sql.SQL(", ").join(sql.Identifier(c) for c in cols_list) + + placeholders = sql.SQL("({})").format( + sql.SQL(", ").join((sql.Placeholder() * len(cols_list))) + ) + + mogrified_values = ", ".join( + client_cur.mogrify(placeholders, row) for row in upsert_tuples + ) + + insert_sql = sql.SQL( + "INSERT INTO {table} ({cols}) VALUES {values}" + ).format( + table=table_ident, + cols=cols_ident, + values=sql.SQL(mogrified_values), + ) + + if simple_primary_key: + conflict_target = sql.SQL("({})").format( + sql.Identifier(keys_list[0]) + ) + else: + conflict_target = sql.SQL("({})").format( + sql.SQL(", ").join(sql.Identifier(k) for k in keys_list) + ) + + if tr_task.insert_only: + conflict_action = sql.SQL("DO NOTHING") + else: + set_clause = sql.SQL(", ").join( + sql.SQL("{col} = EXCLUDED.{col}").format(col=sql.Identifier(c)) + for c in cols_list + ) + conflict_action = sql.SQL("DO UPDATE SET {}").format(set_clause) + + full_upsert_sql = sql.SQL( + "{insert} ON CONFLICT {target} {action};" + ).format( + insert=insert_sql, + target=conflict_target, + action=conflict_action, + ) + + # nosemgrep + cur.execute(full_upsert_sql) + total_upserted[divergent_node] = cur.rowcount if cur.rowcount else 0 + if tr_task.generate_report: + # Ensure this key exists even if upserts were skipped + if divergent_node not in report["changes"]: + report["changes"][divergent_node] = {} report["changes"][divergent_node]["upserted_rows"] = [ dict(zip(cols_list, tup)) for tup in upsert_tuples ] @@ -1250,7 +1504,7 @@ def generate_report(): conn, ( tr_task.client_role - if config.USE_CERT_AUTH and tr_task.invoke_method == "api" + if (config.USE_CERT_AUTH or tr_task.invoke_method == "api") else None ), ) @@ -1332,13 +1586,13 @@ def compare_values(val1: dict, val2: dict) -> bool: for node in other_nodes: if tr_task.insert_only: util.message( - f"{node} INSERTED = {len(full_rows_to_upsert[node])} rows", + f"{node} INSERTED = {total_upserted[node]} rows", p_state="info", quiet_mode=tr_task.quiet_mode, ) else: util.message( - f"{node} UPSERTED = {len(full_rows_to_upsert[node])} rows", + f"{node} UPSERTED = {total_upserted[node]} rows", p_state="info", quiet_mode=tr_task.quiet_mode, ) @@ -1348,7 +1602,7 @@ def compare_values(val1: dict, val2: dict) -> bool: if not tr_task.upsert_only and not tr_task.insert_only: for node in other_nodes: util.message( - f"{node} DELETED = {len(full_rows_to_delete[node])} rows", + f"{node} DELETED = {total_deleted[node]} rows", p_state="info", quiet_mode=tr_task.quiet_mode, ) @@ -1432,11 +1686,9 @@ def table_repair_fix_nulls(tr_task: TableRepairTask) -> None: node_hostname = tr_task.fields.host_map[hostname_key] _, conn = tr_task.connection_pool.get_cluster_node_connection( node_info, - tr_task.cluster_name, - invoke_method=tr_task.invoke_method, client_role=( tr_task.client_role - if config.USE_CERT_AUTH and tr_task.invoke_method == "api" + if (config.USE_CERT_AUTH or tr_task.invoke_method == "api") else None ), drop_privileges=False, @@ -1541,7 +1793,7 @@ def get_key_tuple(row): conn, client_role=( tr_task.client_role - if config.USE_CERT_AUTH and tr_task.invoke_method == "api" + if (config.USE_CERT_AUTH or tr_task.invoke_method == "api") else None ), ) @@ -1736,7 +1988,7 @@ def get_key_tuple(row): conn, client_role=( tr_task.client_role - if config.USE_CERT_AUTH and tr_task.invoke_method == "api" + if (config.USE_CERT_AUTH or tr_task.invoke_method == "api") else None ), ) @@ -1812,11 +2064,9 @@ def table_repair_bidirectional(tr_task: TableRepairTask) -> None: node_hostname = tr_task.fields.host_map[hostname_key] _, conn = tr_task.connection_pool.get_cluster_node_connection( node_info, - tr_task.cluster_name, - invoke_method=tr_task.invoke_method, client_role=( tr_task.client_role - if config.USE_CERT_AUTH and tr_task.invoke_method == "api" + if (config.USE_CERT_AUTH or tr_task.invoke_method == "api") else None ), drop_privileges=False, @@ -1930,7 +2180,7 @@ def perform_inserts(target_node, conn, insert_rows): conn, client_role=( tr_task.client_role - if config.USE_CERT_AUTH and tr_task.invoke_method == "api" + if (config.USE_CERT_AUTH or tr_task.invoke_method == "api") else None ), ) @@ -1971,7 +2221,7 @@ def perform_inserts(target_node, conn, insert_rows): conn, ( tr_task.client_role - if config.USE_CERT_AUTH and tr_task.invoke_method == "api" + if (config.USE_CERT_AUTH or tr_task.invoke_method == "api") else None ), ) @@ -2066,7 +2316,7 @@ def table_rerun_temptable(td_task: TableDiffTask) -> None: table_qry = ( f'CREATE TABLE {temp_table_name} AS SELECT * FROM "{schema}"."{table}" WHERE ' ) - table_qry += generate_where_clause(key, diff_keys) + table_qry += generate_where_clause(key, [diff_keys]) clean_qry = f"DROP TABLE {temp_table_name}" if len(key) == 1: @@ -2091,11 +2341,9 @@ def table_rerun_temptable(td_task: TableDiffTask) -> None: } _, conn = td_task.connection_pool.get_cluster_node_connection( node_info, - td_task.cluster_name, - invoke_method=td_task.invoke_method, client_role=( td_task.client_role - if config.USE_CERT_AUTH and td_task.invoke_method == "api" + if (config.USE_CERT_AUTH or td_task.invoke_method == "api") else None ), ) @@ -2156,11 +2404,9 @@ def table_rerun_temptable(td_task: TableDiffTask) -> None: _, conn = td_task.connection_pool.get_cluster_node_connection( node_info, - td_task.cluster_name, - invoke_method=td_task.invoke_method, client_role=( td_task.client_role - if config.USE_CERT_AUTH and td_task.invoke_method == "api" + if (config.USE_CERT_AUTH or td_task.invoke_method == "api") else None ), ) @@ -2255,28 +2501,14 @@ def table_rerun_async(td_task: TableDiffTask) -> None: cols_list = td_task.fields.cols.split(",") cols_list = [col for col in cols_list if not col.startswith("_Spock_")] - result_queue = Manager().list() - diff_dict = Manager().dict() - row_diff_count = Manager().Value("I", 0) - lock = Manager().Lock() + diff_dict = {} + stop_event = Manager().Event() # Shared variables needed by all workers shared_objects = { - "cluster_name": td_task.cluster_name, - "database": td_task.fields.database, - "node_list": td_task.fields.node_list, - "schema_name": td_task.fields.l_schema, - "table_name": td_task.fields.l_table, - "cols_list": cols_list, - "p_key": td_task.fields.key, - "block_rows": td_task.block_rows, - "simple_primary_key": simple_primary_key, "mode": "rerun", - "result_queue": result_queue, - "diff_dict": diff_dict, - "row_diff_count": row_diff_count, - "lock": lock, - "td_task": td_task, + "stop_event": stop_event, + "task": td_task, } util.message( @@ -2289,13 +2521,14 @@ def table_rerun_async(td_task: TableDiffTask) -> None: diffs_exceeded = False errors = False errors_list = [] + total_diffs = 0 try: with WorkerPool( n_jobs=procs, shared_objects=shared_objects, use_worker_state=True, - use_dill=True, + pass_worker_id=True, ) as pool: for result in pool.imap_unordered( compare_checksums, @@ -2306,15 +2539,36 @@ def table_rerun_async(td_task: TableDiffTask) -> None: iterable_len=len(blocks), progress_bar_style="rich", ): - if result == config.MAX_DIFFS_EXCEEDED: - diffs_exceeded = True - mismatch = True - break - elif result == config.BLOCK_ERROR: + if result["status"] == BLOCK_ERROR: errors = True - errors_list.append(result) + errors_list.append( + { + "node_pair": result["node_pair"], + "batch": result["batch"], + "errors": result["errors"], + } + ) + pool.terminate() # Stop all workers on error break + if result["status"] == BLOCK_MISMATCH: + mismatch = True + # Update the diff dictionary with this worker's results + for node_pair, node_diffs in result["diffs"].items(): + if node_pair not in diff_dict: + diff_dict[node_pair] = {} + for node, diffs in node_diffs.items(): + if node not in diff_dict[node_pair]: + diff_dict[node_pair][node] = [] + diff_dict[node_pair][node].extend(diffs) + + # Update total diffs and check if exceeded + total_diffs += result["total_diffs"] + if total_diffs >= config.MAX_DIFF_ROWS: + diffs_exceeded = True + stop_event.set() # Signal workers to stop + break + if diffs_exceeded: util.message( "Prematurely terminated jobs since diffs have" @@ -2327,10 +2581,6 @@ def table_rerun_async(td_task: TableDiffTask) -> None: ace.handle_task_exception(td_task, context) raise e - for result in result_queue: - if result["status_code"] == config.BLOCK_MISMATCH: - mismatch = True - if errors: context = { "total_rows": total_rows, @@ -2417,7 +2667,6 @@ def table_rerun_async(td_task: TableDiffTask) -> None: p_state="info", quiet_mode=td_task.quiet_mode, ) - ace_db.update_ace_task(td_task) @@ -2460,13 +2709,11 @@ def multi_table_diff(task: Union[RepsetDiffTask, SchemaDiffTask]) -> None: _dbname=task._dbname, fields=task.fields, quiet_mode=task.quiet_mode, - block_rows=getattr(task, "block_rows", config.BLOCK_ROWS_DEFAULT), - max_cpu_ratio=getattr( - task, "max_cpu_ratio", config.MAX_CPU_RATIO_DEFAULT - ), + block_size=getattr(task, "block_size", config.DIFF_BLOCK_SIZE), + max_cpu_ratio=getattr(task, "max_cpu_ratio", config.MAX_CPU_RATIO), output=getattr(task, "output", "json"), _nodes=getattr(task, "_nodes", "all"), - batch_size=getattr(task, "batch_size", config.BATCH_SIZE_DEFAULT), + batch_size=getattr(task, "batch_size", config.DIFF_BATCH_SIZE), skip_db_update=True, table_filter=None, ) @@ -2530,11 +2777,9 @@ def spock_diff(sd_task: SpockDiffTask) -> None: } _, conn = sd_task.connection_pool.get_cluster_node_connection( node_info, - sd_task.cluster_name, - invoke_method=sd_task.invoke_method, client_role=( sd_task.client_role - if config.USE_CERT_AUTH and sd_task.invoke_method == "api" + if (config.USE_CERT_AUTH or sd_task.invoke_method == "api") else None ), ) @@ -2546,7 +2791,7 @@ def spock_diff(sd_task: SpockDiffTask) -> None: try: for cluster_node in sd_task.fields.cluster_nodes: - cur = conns[cluster_node["name"]].cursor() + cur = conns[cluster_node["name"]].cursor(row_factory=dict_row) if ( sd_task.fields.node_list @@ -2635,17 +2880,19 @@ def spock_diff(sd_task: SpockDiffTask) -> None: if table_info == []: hints.append("Hint: No tables in database") for table in table_info: - if table["set_name"] is None: + if table.get("set_name") is None: print(" - Not in a replication set") hints.append( "Hint: Tables not in replication set might not have" " primary keys, or you need to run repset-add-table" ) else: - print(" - " + table["set_name"]) + print(" - " + table.get("set_name")) - diff_spock["rep_set_info"].append({table["set_name"]: table["relname"]}) - print(" - ", table["relname"]) + diff_spock["rep_set_info"].append( + {table.get("set_name"): table.get("relname")} + ) + print(" - ", table.get("relname")) diff_spock["hints"] = hints compare_spock.append(diff_spock) @@ -2666,6 +2913,10 @@ def spock_diff(sd_task: SpockDiffTask) -> None: print("~~~~~~~~~~~~~~~~~~~~~~~~~") for n in range(1, len(compare_spock)): + + if compare_spock[n].get("node") is None: + continue + diff_key = compare_spock[0]["node"] + "/" + compare_spock[n]["node"] if compare_spock[0]["rep_set_info"] == compare_spock[n]["rep_set_info"]: task_context["diffs"][diff_key] = { @@ -2714,11 +2965,9 @@ def schema_diff_objects(sc_task: SchemaDiffTask) -> None: for node in sc_task.fields.cluster_nodes: _, conn = sc_task.connection_pool.get_cluster_node_connection( node, - sc_task.cluster_name, - invoke_method=sc_task.invoke_method, client_role=( sc_task.client_role - if config.USE_CERT_AUTH and sc_task.invoke_method == "api" + if (config.USE_CERT_AUTH or sc_task.invoke_method == "api") else None ), ) @@ -3296,9 +3545,7 @@ def auto_repair(): for node in cluster_nodes: try: - _, conn = ar_task.connection_pool.get_cluster_node_connection( - node, ar_task.cluster_name - ) + _, conn = ar_task.connection_pool.get_cluster_node_connection(node) conn_map[node["name"]] = conn cur = conn_map[node["name"]].cursor(row_factory=dict_row) cur.execute(oid_sql) diff --git a/cli/scripts/ace_daemon.py b/cli/scripts/ace_daemon.py index 35d90eb1..3144b390 100644 --- a/cli/scripts/ace_daemon.py +++ b/cli/scripts/ace_daemon.py @@ -43,7 +43,7 @@ - cluster_name (required): Name of the cluster - table_name (required): Name of the table to diff - dbname (optional): Name of the database -- block_rows (optional): Number of rows per block (default: config.BLOCK_ROWS_DEFAULT) +- block_size (optional): Number of rows per block (default: config.DIFF_BLOCK_SIZE) - max_cpu_ratio (optional): Max CPU usage ratio (default: config.MAX_CPU_RATIO_DEFAULT) - output (optional): Output format, default is 'json' - nodes (optional): Nodes to include in diff, default is 'all' @@ -67,11 +67,11 @@ def table_diff_api(): cluster_name = data.get("cluster_name") table_name = data.get("table_name") dbname = data.get("dbname") - block_rows = data.get("block_rows", config.BLOCK_ROWS_DEFAULT) - max_cpu_ratio = data.get("max_cpu_ratio", config.MAX_CPU_RATIO_DEFAULT) + block_size = data.get("block_size", config.DIFF_BLOCK_SIZE) + max_cpu_ratio = data.get("max_cpu_ratio", config.MAX_CPU_RATIO) output = data.get("output", "json") nodes = data.get("nodes", "all") - batch_size = data.get("batch_size", config.BATCH_SIZE_DEFAULT) + batch_size = data.get("batch_size", config.DIFF_BATCH_SIZE) table_filter = data.get("table_filter") quiet = data.get("quiet", False) @@ -88,7 +88,7 @@ def table_diff_api(): cluster_name=cluster_name, _table_name=table_name, _dbname=dbname, - block_rows=block_rows, + block_size=block_size, max_cpu_ratio=max_cpu_ratio, output=output, _nodes=nodes, @@ -237,9 +237,6 @@ def table_repair_api(): table_name (str): Name of the table to rerun the diff on (required) dbname (str): Name of the database (optional) quiet (bool): Whether to suppress output (optional, default: False) - behavior (str): The behavior to use for rerunning - (optional, default: "multiprocessing") - Supported values: "multiprocessing", "hostdb" Returns: JSON response with task_id and submitted_at timestamp on success, @@ -260,7 +257,6 @@ def table_rerun_api(): table_name = data.get("table_name") dbname = data.get("dbname") quiet = data.get("quiet", False) - behavior = data.get("behavior", "multiprocessing") if not cluster_name or not diff_file or not table_name: return ( @@ -280,11 +276,11 @@ def table_rerun_api(): cluster_name=cluster_name, _table_name=table_name, _dbname=dbname, - block_rows=config.BLOCK_ROWS_DEFAULT, - max_cpu_ratio=config.MAX_CPU_RATIO_DEFAULT, + block_size=config.DIFF_BLOCK_SIZE, + max_cpu_ratio=config.MAX_CPU_RATIO, output="json", _nodes="all", - batch_size=config.BATCH_SIZE_DEFAULT, + batch_size=config.DIFF_BATCH_SIZE, quiet_mode=quiet, diff_file_path=diff_file, invoke_method="api", @@ -302,26 +298,14 @@ def table_rerun_api(): return jsonify({"error": str(e)}), 400 try: - if behavior == "multiprocessing": - scheduler.add_job(ace_core.table_rerun_async, args=(raw_args,)) - now = datetime.now() - return jsonify( - { - "task_id": task_id, - "submitted_at": now.isoformat(), - } - ) - elif behavior == "hostdb": - scheduler.add_job(ace_core.table_rerun_temptable, args=(raw_args,)) - now = datetime.now() - return jsonify( - { - "task_id": task_id, - "submitted_at": now.isoformat(), - } - ) - else: - return jsonify({"error": f"Invalid behavior: {behavior}"}), 400 + scheduler.add_job(ace_core.table_rerun_temptable, args=(raw_args,)) + now = datetime.now() + return jsonify( + { + "task_id": task_id, + "submitted_at": now.isoformat(), + } + ) except Exception as e: return jsonify({"error": str(e)}), 400 @@ -333,7 +317,7 @@ def table_rerun_api(): cluster_name (str): Name of the cluster (required) repset_name (str): Name of the repset to diff (required) dbname (str): Name of the database (optional) - block_rows (int): Number of rows per block (default: config.BLOCK_ROWS_DEFAULT) + block_size (int): Number of rows per block (default: config.DIFF_BLOCK_SIZE) max_cpu_ratio (float): Maximum CPU usage ratio (default: config.MAX_CPU_RATIO_DEFAULT) output (str): Output format (default: "json") @@ -360,11 +344,11 @@ def repset_diff_api(): cluster_name = data.get("cluster_name") repset_name = data.get("repset_name") dbname = data.get("dbname") - block_rows = data.get("block_rows", config.BLOCK_ROWS_DEFAULT) - max_cpu_ratio = data.get("max_cpu_ratio", config.MAX_CPU_RATIO_DEFAULT) + block_size = data.get("block_size", config.DIFF_BLOCK_SIZE) + max_cpu_ratio = data.get("max_cpu_ratio", config.MAX_CPU_RATIO) output = data.get("output", "json") nodes = data.get("nodes", "all") - batch_size = data.get("batch_size", config.BATCH_SIZE_DEFAULT) + batch_size = data.get("batch_size", config.DIFF_BATCH_SIZE) quiet = data.get("quiet", False) skip_tables = data.get("skip_tables") skip_file = data.get("skip_file") @@ -382,7 +366,7 @@ def repset_diff_api(): cluster_name=cluster_name, _dbname=dbname, repset_name=repset_name, - block_rows=block_rows, + block_size=block_size, max_cpu_ratio=max_cpu_ratio, output=output, _nodes=nodes, @@ -952,7 +936,7 @@ def create_schedules(): # Define valid parameters for each job type valid_table_diff_params = { "dbname", - "block_rows", + "block_size", "max_cpu_ratio", "output", "nodes", diff --git a/cli/scripts/ace_data_models.py b/cli/scripts/ace_data_models.py index 594f8aab..8c83128f 100644 --- a/cli/scripts/ace_data_models.py +++ b/cli/scripts/ace_data_models.py @@ -34,6 +34,7 @@ class DerivedFields: host_map: dict = None table_list: list = None col_types: dict = None + simple_primary_key: bool = None @dataclass @@ -45,13 +46,16 @@ class TableDiffTask: # User-specified, validated fields cluster_name: str # Required - block_rows: int + block_size: int max_cpu_ratio: float output: str batch_size: int table_filter: str quiet_mode: bool + mode: str = "diff" + _override_block_size: bool = False + # For table-diff, the diff_file_path is # obtained after the run of table-diff, # and is not mandatory @@ -131,7 +135,7 @@ class RepsetDiffTask: # Optional fields # Non-default members since the handler method will fill in the # default values - block_rows: int + block_size: int max_cpu_ratio: float output: str batch_size: int @@ -249,3 +253,39 @@ class AutoRepairTask: # Task-specific parameters scheduler: Task = field(default_factory=Task) + + +@dataclass +class MerkleTreeTask: + # Can be one of : build, update, rebalance + mode: str + cluster_name: str + _table_name: str + _dbname: str + _nodes: str + + analyse: bool + rebalance: bool + recreate_objects: bool + block_size: int + max_cpu_ratio: float + batch_size: int + output: str + quiet_mode: bool + + ranges: list = None + write_ranges: bool = False + ranges_file: str = None + row_estimate: int = 0 + invoke_method: str = "cli" + + # Client role from certificate CN when invoked via API + client_role: str = None + + connection_pool: ConnectionPool = field(default_factory=ConnectionPool) + diff_summary: dict = field(default_factory=dict) + + scheduler: Task = field(default_factory=Task) + + # Derived fields + fields: DerivedFields = field(default_factory=DerivedFields) diff --git a/cli/scripts/ace_mtree.py b/cli/scripts/ace_mtree.py new file mode 100644 index 00000000..a621b66b --- /dev/null +++ b/cli/scripts/ace_mtree.py @@ -0,0 +1,2805 @@ +from concurrent.futures import ThreadPoolExecutor +from itertools import combinations, zip_longest +import json +from multiprocessing import Manager +from datetime import datetime +import traceback +import os +from typing import Tuple, Any +from dataclasses import dataclass + +from mpire import WorkerPool +from mpire.utils import make_single_arguments +from psycopg import sql +from psycopg import IsolationLevel +from psycopg.types.composite import register_composite, CompositeInfo +from tqdm import tqdm + +import ace_html_reporter +import util +import ace +import ace_core +import ace_db +import ace_config as config +from ace_data_models import MerkleTreeTask +from ace_exceptions import AceException +from ace_constants import BLOCK_OK, BLOCK_MISMATCH, BLOCK_ERROR +from ace_sql import ( + CREATE_BULK_TRIGGER_FUNCTION, + CREATE_METADATA_TABLE, + CREATE_XOR_FUNCTION, + DROP_BULK_TRIGGER_FUNCTION, + DROP_METADATA_TABLE, + DROP_XOR_FUNCTION, + DROP_MTREE_TABLE, + DROP_MTREE_TRIGGERS, + ENABLE_ALWAYS, + ESTIMATE_ROW_COUNT, + GET_BLOCK_COUNT_COMPOSITE, + GET_BLOCK_COUNT_SIMPLE, + GET_BLOCK_SIZE_FROM_METADATA, + GET_COUNT_COMPOSITE, + GET_COUNT_SIMPLE, + GET_MAX_VAL_SIMPLE, + GET_PKEY_TYPE, + GET_ROW_COUNT_ESTIMATE, + GET_MAX_VAL_COMPOSITE, + GET_SPLIT_POINT_COMPOSITE, + GET_SPLIT_POINT_SIMPLE, + UPDATE_LEAF_HASHES, + UPDATE_MAX_VAL, + UPDATE_METADATA, + COMPUTE_LEAF_HASHES, + BUILD_PARENT_NODES, + INSERT_BLOCK_RANGES, + GET_DIRTY_AND_NEW_BLOCKS, + CLEAR_DIRTY_FLAGS, + GET_NODE_CHILDREN, + GET_ROOT_NODE, + GET_LEAF_RANGES, + CREATE_GENERIC_TRIGGER, + INSERT_COMPOSITE_BLOCK_RANGES, + CREATE_SIMPLE_MTREE_TABLE, + CREATE_COMPOSITE_MTREE_TABLE, + DELETE_PARENT_NODES, + GET_MAX_NODE_POSITION, + UPDATE_BLOCK_RANGE_END, + DELETE_BLOCK, + UPDATE_NODE_POSITIONS_SEQUENTIAL, + FIND_BLOCKS_TO_SPLIT, + FIND_BLOCKS_TO_MERGE_COMPOSITE, + FIND_BLOCKS_TO_MERGE_SIMPLE, + GET_MAX_NODE_LEVEL, + COMPARE_BLOCKS_SQL, + UPDATE_NODE_POSITIONS_TEMP, +) + + +@dataclass +class BlockBoundary: + """Represents a block boundary for parallel processing""" + + block_id: int + block_start: Tuple[Any, ...] + block_end: Tuple[Any, ...] + + +def safe_min(a, b): + if a is None and b is None: + return None + if a is None: + return b + if b is None: + return a + return min(a, b) + + +def safe_max(a, b): + if a is None and b is None: + return None + if a is None: + return b + if b is None: + return a + return max(a, b) + + +def init_hash_conn_pool(worker_id, shared_objects, worker_state): + """ + Initialise connection pool for hash computation workers + + Args: + shared_objects: The shared objects from the mpire worker pool manager. + contains the node and task objects, but might optimise this later. + + worker_state: The empty worker state dictionary for initialisation. + This function is called once per worker, and populates the worker state + with the connection and cursor for the worker. + """ + node = shared_objects["node"] + task = shared_objects["task"] + + try: + _, conn = task.connection_pool.connect(node) + task.connection_pool.drop_privileges( + conn, + client_role=( + task.client_role + if config.USE_CERT_AUTH or task.invoke_method == "api" + else None + ), + ) + worker_state["conn"] = conn + worker_state["cur"] = conn.cursor() + + except Exception as e: + if "conn" in worker_state: + try: + worker_state["conn"].close() + except Exception: + pass + raise AceException(f"Error initializing worker connection: {str(e)}") + + +def close_hash_conn_pool(worker_id, shared_objects, worker_state): + """ + Close connection pool for hash computation workers + + Args: + shared_objects: The shared objects from the mpire worker pool manager. + Not needed here, but retained because mpire requires it. + + worker_state: The worker state after the worker initialisation. Contains + the connection and cursor for the worker. + """ + try: + worker_state["conn"].commit() + worker_state["cur"].close() + worker_state["conn"].close() + except Exception as e: + raise AceException(f"Error closing worker connection: {str(e)}") + + +def compute_block_hashes(worker_id, shared_objects, worker_state, args): + """ + Worker function that computes hashes for a block + + Args: + shared_objects: The shared objects from the mpire worker pool manager. + contains the node and task objects. + + worker_state: The worker state after the worker initialisation. Contains + the connection and cursor for the worker. + + args: The arguments for the worker function. Contains the block boundary + and the SQL query to compute the hashes. + """ + block = args + task = shared_objects["task"] + + try: + conn = worker_state.get("conn") + cur = conn.cursor() + + key_columns = task.fields.key.split(",") + is_composite = len(key_columns) > 1 + + if is_composite: + where_conditions = [] + + pkey_cols = sql.SQL(", ").join(sql.Identifier(col) for col in key_columns) + where_conditions.append( + sql.SQL("({pkey_cols}) >= ({pkey_values})").format( + pkey_cols=pkey_cols, + pkey_values=sql.SQL(", ").join( + sql.Literal(val) for val in block.block_start + ), + ) + ) + where_conditions.append( + sql.SQL("({pkey_cols}) <= ({pkey_values})").format( + pkey_cols=pkey_cols, + pkey_values=sql.SQL(", ").join( + sql.Literal(val) for val in block.block_end + ), + ) + ) + + # TODO: Double check this + if not where_conditions: + where_clause = sql.SQL("TRUE") + else: + where_clause = sql.SQL(" AND ").join(where_conditions) + + else: + where_conditions = [] + + where_conditions.append( + sql.SQL("{pkey_col} >= {pkey_value}").format( + pkey_col=sql.Identifier(task.fields.key), + pkey_value=sql.Literal(*block.block_start), + ) + ) + + where_conditions.append( + sql.SQL("{pkey_col} <= {pkey_value}").format( + pkey_col=sql.Identifier(task.fields.key), + pkey_value=sql.Literal(*block.block_end), + ) + ) + + if not where_conditions: + where_clause = sql.SQL("TRUE") + else: + where_clause = sql.SQL(" AND ").join(where_conditions) + + cur.execute( + sql.SQL(COMPUTE_LEAF_HASHES).format( + schema=sql.Identifier(task.fields.l_schema), + table=sql.Identifier(task.fields.l_table), + key=sql.SQL(", ").join(sql.Identifier(col) for col in key_columns), + columns=sql.SQL(", ").join( + sql.Identifier(col) for col in task.fields.cols.split(",") + ), + where_clause=where_clause, + ) + ) + + leaf_hash = cur.fetchone()[0] + + return {"node_position": block.block_id, "leaf_hash": leaf_hash} + + except Exception as e: + print(f"Error computing hash for block {block.block_id}: {str(e)}") + traceback.print_exc() + return None + + +def create_mtree_objects( + conn, schema, table, key, total_rows, block_size, num_blocks, recreate_objects=False +): + """ + Creates the necessary database objects for Merkle tree implementation + + Args: + conn: Database connection + schema: Schema name + table: Table name + key: Primary key column(s) + total_rows: Estimated total number of rows + num_blocks: Number of blocks to split the table into + recreate_objects: Whether to drop and recreate all objects + """ + key_columns = key.split(",") + is_composite = len(key_columns) > 1 + + with conn.cursor() as cur: + if recreate_objects: + _mtree_init(conn) + + cur.execute( + sql.SQL("DROP TABLE IF EXISTS {mtree_table}").format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}") + ) + ) + + if is_composite: + key_type_columns = [] + for col in key_columns: + cur.execute( + sql.SQL(GET_PKEY_TYPE).format( + schema=sql.Literal(schema), + table=sql.Literal(table), + key=sql.Literal(col), + ) + ) + col_type = cur.fetchone()[0] + key_type_columns.append(f"{col} {col_type}") + + cur.execute( + sql.SQL(CREATE_COMPOSITE_MTREE_TABLE).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + range_idx=sql.SQL(f"idx_range_{schema}_{table}"), + schema=sql.SQL(schema), + table=sql.SQL(table), + key_type_columns=sql.SQL(", ").join( + sql.SQL(col) for col in key_type_columns + ), + ) + ) + else: + cur.execute( + sql.SQL(GET_PKEY_TYPE).format( + schema=sql.Literal(schema), + table=sql.Literal(table), + key=sql.Literal(key), + ) + ) + pkey_type = cur.fetchone()[0] + + cur.execute( + sql.SQL(CREATE_SIMPLE_MTREE_TABLE).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + pkey_type=sql.SQL(pkey_type), + range_idx=sql.Identifier(f"idx_range_{schema}_{table}"), + ) + ) + + cur.execute( + sql.SQL(CREATE_GENERIC_TRIGGER).format( + trigger=sql.SQL(f"trg_mtree_{schema}_{table}"), + schema=sql.Identifier(schema), + table=sql.Identifier(table), + key=sql.Literal(key), + ) + ) + + cur.execute( + sql.SQL(ENABLE_ALWAYS).format( + schema=sql.Identifier(schema), + table=sql.Identifier(table), + insert_trigger=sql.SQL(f"trg_mtree_{schema}_{table}_insert_stmt"), + update_trigger=sql.SQL(f"trg_mtree_{schema}_{table}_update_stmt"), + delete_trigger=sql.SQL(f"trg_mtree_{schema}_{table}_delete_stmt"), + ) + ) + + cur.execute( + UPDATE_METADATA, + (schema, table, total_rows, block_size, num_blocks, is_composite), + ) + + conn.commit() + + +def get_row_estimate(conn, schema, table, block_size, analyse=False): + """ + We cannot use a naïve count(*) for merkle tree candidates because the + table is too large. Instead, we use a multi-step process to get an estimate: + + 1. We first check the live_tuples in pg_stat_user_tables. + 2. If 1 is not available, we use the reltuples in pg_class. + 3. If neither is available, we use the pg_relation_size / 8192 * 0.7. + 8192 is the page size used by PostgreSQL in bytes, and 0.7 is the + approximate fill factor of the page. + + Args: + conn: The database connection + schema: The schema of the target table + table: The target table + analyse: Whether to analyse the table + """ + + cur = conn.cursor() + + if analyse: + cur.execute("BEGIN") + cur.execute("SET parallel_tuple_cost = 0") + cur.execute("SET parallel_setup_cost = 0") + + print(f"Analysing {schema}.{table} on node {conn.info.host}") + print("This might take a while...") + + # TODO: This might take a while so need a different strategy here + cur.execute( + sql.SQL("ANALYZE {schema}.{table}").format( + schema=sql.Identifier(schema), table=sql.Identifier(table) + ) + ) + + cur.execute( + sql.SQL(ESTIMATE_ROW_COUNT).format( + schema=sql.Literal(schema), + table=sql.Literal(table), + ), + ) + total_rows = cur.fetchone()[0] + num_blocks = (total_rows - 1) // block_size + 1 + + conn.commit() + return total_rows, num_blocks + + +def process_block_ranges(offsets: list): + + block_ranges = [] + + del offsets[-1] + + if len(offsets) == 2: + for i, offset in enumerate(offsets): + # Yes, this needs to be a tuple to maintain consistency + block_ranges.append((i, (offset[0],), (offset[1],))) + else: + for i, offset in enumerate(offsets): + rng = ( + i, + tuple([offset[k] for k in range(len(offset)) if k % 2 == 0]), + tuple([offset[k] for k in range(len(offset)) if k % 2 == 1]), + ) + + block_ranges.append(rng) + + return block_ranges + + +def build_mtree(mtree_task: MerkleTreeTask) -> None: + """ + Build a Merkle tree for a table using parallel processing. + The tree is stored in a separate table for each source table. + Parent nodes are computed using XOR of child hashes. + + Args: + cluster_name (str): Name of the cluster + table_name (str): Name of the table to build tree for + dbname (str, optional): Database name. Defaults to the first entry + in the databases array of the cluster json file. + """ + + ace.merkle_tree_checks(mtree_task, skip_validation=True) + total_rows = 0 + try: + # First we need to get the row estimates and blocks from all nodes + # before we can compute the block ranges. + # The block range computation will happen on just one node, and the same + # ranges will be used for all nodes. + # This is crucial because otherwise, the leaf node hashes are meaningless + # if they correspond to different block ranges. + + max_blocks = 0 + ref_node = None + schema = mtree_task.fields.l_schema + table = mtree_task.fields.l_table + key = mtree_task.fields.key + num_blocks = 0 + block_ranges = mtree_task.ranges + + if not block_ranges: + for node in mtree_task.fields.cluster_nodes: + _, conn = mtree_task.connection_pool.get_cluster_node_connection(node) + try: + total_rows, num_blocks = get_row_estimate( + conn, + schema, + table, + mtree_task.block_size, + analyse=mtree_task.analyse, + ) + + if num_blocks > max_blocks: + max_blocks = num_blocks + ref_node = node + except Exception as e: + conn.rollback() + traceback.print_exc() + raise AceException( + f"Error creating mtree objects on {node['name']}: {str(e)}" + ) + + print(f"Using node {ref_node['name']} as the reference node") + + block_ranges = [] + + # Now let's compute the block ranges on the reference node + msg = ( + f"Calculating block ranges for {max_blocks:,} blocks " + f"(~{total_rows:,} rows)" + ) + print(msg) + + sample_method, sample_percent = ace.compute_sampling_parameters(total_rows) + + print(f"Using {sample_method} with sample percent {sample_percent}") + + _, conn = mtree_task.connection_pool.get_cluster_node_connection(ref_node) + + offsets_query = ace.generate_pkey_offsets_query( + schema=schema, + table=table, + key_columns=key.split(","), + table_sample_method=sample_method, + sample_percent=sample_percent, + ntile_count=max_blocks, + ) + + with conn.cursor() as cur: + cur.execute(offsets_query) + offsets = cur.fetchall() + block_ranges = process_block_ranges(offsets) + + if mtree_task.write_ranges: + now = datetime.now().strftime("%Y%m%d_%H%M%S") + filename = f"{now}_{schema}_{table}_ranges.json" + with open(filename, "w") as f: + json.dump(block_ranges, f, default=str) + + print(f"\nBlock ranges written to {util.set_colour(filename, 'blue')}") + + # We're ready to build the merkle tree + schema_table = f"{schema}.{table}" + print(f"\nBuilding merkle tree for {schema_table}") + + for node in mtree_task.fields.cluster_nodes: + print(f"\nProcessing node: {node['name']}") + _, conn = mtree_task.connection_pool.get_cluster_node_connection(node) + + recreate_objects = check_if_init_needed( + conn, schema, "bulk_block_tracking_dispatcher" + ) + + create_mtree_objects( + conn, + schema, + table, + key, + total_rows, + mtree_task.block_size, + num_blocks, + recreate_objects=recreate_objects, + ) + + try: + cur = conn.cursor() + + key_columns = key.split(",") + is_composite = len(key_columns) > 1 + + if is_composite: + insert_sql = sql.SQL(INSERT_COMPOSITE_BLOCK_RANGES).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + start_tuple_values=sql.SQL(", ").join( + [sql.Placeholder()] * len(key_columns) + ), + end_tuple_values=sql.SQL(", ").join( + [sql.Placeholder()] * len(key_columns) + ), + ) + + cur.executemany( + insert_sql, + [ + (id, *start_tuple, *end_tuple) + for id, start_tuple, end_tuple in block_ranges + ], + ) + else: + insert_sql = sql.SQL(INSERT_BLOCK_RANGES).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}") + ) + cur.executemany( + insert_sql, + [(id, *start, *end) for id, start, end in block_ranges], + ) + + conn.commit() + + work_items = [] + for block in block_ranges: + work_items.append( + ( + BlockBoundary( + block_id=block[0], + block_start=block[1], + block_end=block[2], + ) + ) + ) + + max_workers = int(os.cpu_count() * mtree_task.max_cpu_ratio * 2) + n_jobs = min(len(work_items), max_workers) + + shared_objects = { + "task": mtree_task, + "node": node, + } + + results = [] + + cur.close() + conn.close() + mtree_task.connection_pool.close_all() + + with WorkerPool( + n_jobs=n_jobs, + shared_objects=shared_objects, + use_worker_state=True, + pass_worker_id=True, + ) as pool: + results = list( + pool.imap_unordered( + compute_block_hashes, + make_single_arguments(work_items), + iterable_len=len(work_items), + worker_init=init_hash_conn_pool, + worker_exit=close_hash_conn_pool, + progress_bar=True, + progress_bar_options={"desc": "Computing hashes"}, + ) + ) + + failed = [r for r in results if not r] + if failed: + raise AceException( + f"Failed to compute hashes for {len(failed)} blocks" + ) + + _, new_conn = mtree_task.connection_pool.get_cluster_node_connection( + node + ) + parent_cur = new_conn.cursor() + + print("\nUpdating leaf nodes with computed hashes...") + + parent_cur.executemany( + sql.SQL(UPDATE_LEAF_HASHES).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ), + results, + ) + + print("\nBuilding parent nodes...") + + parent_cur.execute( + sql.SQL(DELETE_PARENT_NODES).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}") + ) + ) + + level = 0 + while True: + parent_cur.execute( + sql.SQL(BUILD_PARENT_NODES).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ), + {"node_level": level}, + ) + + count = parent_cur.fetchone()[0] + if count <= 1: + break + level += 1 + + new_conn.commit() + print( + f"Merkle tree built successfully on {node['name']}" + f" with {level + 1} levels" + ) + + conn.close() + + except Exception as e: + conn.rollback() + traceback.print_exc() + raise AceException( + f"Error building Merkle tree on {node['name']}: {str(e)}" + ) + + mtree_task.scheduler.task_status = "COMPLETED" + mtree_task.scheduler.task_context = {"total_rows": total_rows} + except Exception as e: + mtree_task.scheduler.task_status = "FAILED" + mtree_task.scheduler.task_context = {"errors": [str(e)]} + raise e + finally: + mtree_task.scheduler.finished_at = datetime.now() + mtree_task.scheduler.time_taken = util.round_timedelta( + mtree_task.scheduler.finished_at - mtree_task.scheduler.started_at + ).total_seconds() + ace_db.update_ace_task(mtree_task) + mtree_task.connection_pool.close_all() + + +def split_blocks(conn, schema, table, key, blocks, block_size): + """ + Split blocks if they are too large. + Returns a list of block positions that were modified. + """ + try: + cur = conn.cursor() + modified_positions = set() + target_size = block_size + + # First delete all parent nodes since we'll be modifying the tree structure + # This is not expensive at all since computing parent hashes is significantly + # cheaper than computing leaf hashes. + cur.execute( + sql.SQL( + """ + DELETE FROM {mtree_table} + WHERE node_level > 0 + """ + ).format(mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}")) + ) + + key_columns = key.split(",") + is_composite = len(key_columns) > 1 + + if is_composite: + key_types = [] + for col in key_columns: + cur.execute( + sql.SQL(GET_PKEY_TYPE).format( + schema=sql.Literal(schema), + table=sql.Literal(table), + key=sql.Literal(col), + ), + ) + key_types.append(cur.fetchone()[0]) + else: + cur.execute( + sql.SQL(GET_PKEY_TYPE).format( + schema=sql.Literal(schema), + table=sql.Literal(table), + key=sql.Literal(key), + ) + ) + pkey_type = cur.fetchone()[0] + + blocks = sorted(blocks, key=lambda x: x[0]) + i = 0 + except Exception as e: + conn.rollback() + raise AceException(f"Error splitting blocks on {schema}.{table}: {str(e)}") + + pbar = tqdm(total=len(blocks), desc="Processing blocks", leave=False) + + while i < len(blocks): + pos, start, end = blocks[i] + pbar.update(1) + + # When inserts happen after the range_end of the last block, we mark that + # block as dirty and set the range_end to null. So, if we're attempting + # to add new blocks at the end, we need to find the actual max value. + if i == len(blocks) - 1 and end is None: + if is_composite: + pkey_cols = sql.SQL(", ").join( + sql.Identifier(col) for col in key_columns + ) + pkey_values = sql.SQL(", ").join(sql.Literal(val) for val in start) + + cur.execute( + sql.SQL(GET_MAX_VAL_COMPOSITE).format( + schema=sql.Identifier(schema), + table=sql.Identifier(table), + pkey_cols=pkey_cols, + pkey_values=pkey_values, + ) + ) + max_vals = cur.fetchone() + if max_vals is not None: + end = max_vals + cur.execute( + sql.SQL(UPDATE_MAX_VAL).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ), + (end, pos), + ) + else: + cur.execute( + sql.SQL(GET_MAX_VAL_SIMPLE).format( + schema=sql.Identifier(schema), + table=sql.Identifier(table), + key=sql.Identifier(key), + ), + (start,), + ) + max_val = cur.fetchone()[0] + if max_val is not None: + end = max_val + cur.execute( + sql.SQL(UPDATE_MAX_VAL).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ), + (end, pos), + ) + + # Get actual row count for this block + if is_composite: + where_conditions = [] + pkey_cols = sql.SQL(", ").join(sql.Identifier(col) for col in key_columns) + + where_conditions.append( + sql.SQL("({pkey_cols}) >= ({pkey_values})").format( + pkey_cols=pkey_cols, + pkey_values=sql.SQL(", ").join(sql.Literal(val) for val in start), + ) + ) + + if end is not None: + where_conditions.append( + sql.SQL("({pkey_cols}) <= ({pkey_values})").format( + pkey_cols=pkey_cols, + pkey_values=sql.SQL(", ").join(sql.Literal(val) for val in end), + ) + ) + + where_clause = sql.SQL(" AND ").join(where_conditions) + + cur.execute( + sql.SQL(GET_COUNT_COMPOSITE).format( + schema=sql.Identifier(schema), + table=sql.Identifier(table), + where_clause=where_clause, + ) + ) + else: + cur.execute( + sql.SQL(GET_COUNT_SIMPLE).format( + schema=sql.Identifier(schema), + table=sql.Identifier(table), + key=sql.Identifier(key), + pkey_type=sql.SQL(pkey_type), + ), + (start, end, end), + ) + + count = cur.fetchone()[0] + + if count >= target_size * 2: + # Find the split point at the midpoint of the block + if is_composite: + pkey_cols = sql.SQL(", ").join( + sql.Identifier(col) for col in key_columns + ) + order_cols = sql.SQL("({pkey_cols})").format(pkey_cols=pkey_cols) + where_conditions = [] + + where_conditions.append( + sql.SQL("({pkey_cols}) >= ({pkey_values})").format( + pkey_cols=pkey_cols, + pkey_values=sql.SQL(", ").join( + sql.Literal(val) for val in start + ), + ) + ) + + if end is not None: + where_conditions.append( + sql.SQL("({pkey_cols}) <= ({pkey_values})").format( + pkey_cols=pkey_cols, + pkey_values=sql.SQL(", ").join( + sql.Literal(val) for val in end + ), + ) + ) + + where_clause = sql.SQL(" AND ").join(where_conditions) + + cur.execute( + sql.SQL(GET_SPLIT_POINT_COMPOSITE).format( + schema=sql.Identifier(schema), + table=sql.Identifier(table), + pkey_cols=pkey_cols, + where_clause=where_clause, + order_cols=order_cols, + ), + (count // 2,), # Split at midpoint + ) + else: + cur.execute( + sql.SQL(GET_SPLIT_POINT_SIMPLE).format( + schema=sql.Identifier(schema), + table=sql.Identifier(table), + key=sql.Identifier(key), + pkey_type=sql.SQL(pkey_type), + ), + (start, end, end, count // 2), # Split at midpoint + ) + + split_point = cur.fetchone()[0] + + """ + We're going to be inserting the new block at the end to avoid disrupting + the existing parent hashes. Why? Consider this example: + + 0: 1-100: x1 + 1: 101-500: x2 + 2: 501-600: x3 + 3: 601-700: x4 + 4: 701-1000: x5 + + It's ascii tree would look like this: + + p6 + / \ + / \ + / \ + p4 p5 + / \ \ + / \ \ + / \ \ + p1 p2 p3 + / \ / \ \ + x1 x2 x3 x4 x5 + + + Let's assume our initial blocks are all ~100 rows in size. + Now, say many inserts happen in block 101-500, and we need to split it. + If we split block 1, and insert the new block right after it, we get: + + 0: 1-100: x1 + 1: 101-300: m + 2: 301-500: n + 3: 501-600: x3 + 4: 601-700: x4 + 5: 701-1000: x5 + + The new tree would look like this: + m & n indicate new hashes, and primes (') indicate changed hashes + + p6' + / \ + / \ + / \ + / \ + p4' p5' + / \ \ + / \ \ + / \ \ + p1' p2' p3' + / \ / \ / \ + x1 m n x3 x4 x5 + + While doing a table-diff, we'd have to go all the way down to the leaf + nodes to detect which blocks don't match anymore. In other words, + even though just 2 leaf blocks' hashes changed, all their parent hashes + changed unnecessarily. This is would defeat the purpose of having + merkle trees in the first place. + + Instead, if we insert the new block at the end, we retain the + parent-child relationships of existing leaf nodes, and only modify them + where the split happened. + + With this logic, our new tree would look like this: + + 0: 1-100: x1 + 1: 101-300: m + 2: 501-600: x3 + 3: 601-700: x4 + 4: 701-1000: x5 + 5: 301-500: n + + p6' + / \ + / \ + / \ + / \ + p4' p5' + / \ \ + / \ \ + / \ \ + p1' p2 p3' + / \ / \ / \ + x1 m x3 x4 x5 n + + In our example, p2 remained unchanged, but in a larger tree, several + parent nodes could potentially remain unchanged, thereby speeding-up + our diff logic. + """ + + # Get the next available position at the end + cur.execute( + sql.SQL(GET_MAX_NODE_POSITION).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}") + ) + ) + new_pos = cur.fetchone()[0] + + # Create new block at the end + if is_composite: + insert_sql = sql.SQL(INSERT_COMPOSITE_BLOCK_RANGES).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + start_tuple_values=sql.SQL(", ").join( + [sql.Placeholder()] * len(key_columns) + ), + end_tuple_values=sql.SQL(", ").join( + [sql.Placeholder()] * len(key_columns) + ), + ) + cur.execute( + insert_sql, + (new_pos, *split_point, *end), + ) + else: + cur.execute( + sql.SQL(INSERT_BLOCK_RANGES).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ), + (new_pos, split_point, end), + ) + + cur.execute( + sql.SQL(UPDATE_BLOCK_RANGE_END).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}") + ), + (split_point, pos), + ) + + modified_positions.add(pos) + modified_positions.add(new_pos) + + blocks[i] = (pos, start, split_point) + blocks.insert(i + 1, (new_pos, split_point, end)) + i += 1 + continue + + i += 1 + + conn.commit() + pbar.close() + return list(modified_positions) + + +def merge_blocks(conn, schema, table, key, blocks, block_size): + """ + Merge blocks if they are too small. + Returns a list of block positions that were modified. + """ + cur = conn.cursor() + modified_positions = set() + target_size = block_size + + # We will trigger a merge if the actual block size is less than 25% of the + # target size + MERGE_THRESHOLD = 0.25 + + # Whenever there is a merge, the node_positions of leaf nodes + # become invalid. Now, we cannot simply do a node_position = node_position - 1 + # since there might be other leaf nodes at that location. + # So we move all leaf nodes to a temporary position and then update the + # node_positions of the leaf nodes after the rebalancing is done + # This temp offset is large enough for the foreseeable future. + # + # Napkin math: For a conflict to happen with a 10^6 temp offset, and a + # block size of say 10^5, we would need 10^11 rows or 100 billion rows in the + # table, which is a lot. + # + # NOTE: This operation is disruptive. It *will* change parent-child relationships, + # thereby potentially slowing down table-diff. Use --rebalance=true infrequently, + # or during maintenance windows. + TEMP_OFFSET = 1_000_000 + + # First delete all parent nodes since we'll be modifying the tree structure + # This is not expensive at all since computing parent hashes is significantly + # cheaper than computing leaf hashes. + cur.execute( + sql.SQL(DELETE_PARENT_NODES).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}") + ) + ) + + key_columns = key.split(",") + is_composite = len(key_columns) > 1 + + if is_composite: + key_types = [] + for col in key_columns: + cur.execute( + sql.SQL(GET_PKEY_TYPE).format( + schema=sql.Literal(schema), + table=sql.Literal(table), + key=sql.Literal(col), + ), + ) + key_types.append(cur.fetchone()[0]) + else: + cur.execute( + sql.SQL(GET_PKEY_TYPE).format( + schema=sql.Literal(schema), + table=sql.Literal(table), + key=sql.Literal(key), + ), + ) + pkey_type = cur.fetchone()[0] + + blocks = sorted(blocks, key=lambda x: x[0]) + i = 0 + + pbar = tqdm(total=len(blocks), desc="Processing blocks") + + while i < len(blocks): + pos, start, end = blocks[i] + pbar.update(1) + + # When inserts happen after the range_end of the last block, we mark that + # block as dirty and set the range_end to null. So, if we're attempting + # to add new blocks at the end, we need to find the actual max value. + if i == len(blocks) - 1 and end is None: + if is_composite: + pkey_cols = sql.SQL(", ").join( + sql.Identifier(col) for col in key_columns + ) + pkey_values = sql.SQL(", ").join(sql.Literal(val) for val in start) + + cur.execute( + sql.SQL(GET_MAX_VAL_COMPOSITE).format( + schema=sql.Identifier(schema), + table=sql.Identifier(table), + pkey_cols=pkey_cols, + pkey_values=pkey_values, + ) + ) + max_vals = cur.fetchone() + if max_vals is not None: + end = max_vals + cur.execute( + sql.SQL(UPDATE_MAX_VAL).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ), + (end, pos), + ) + else: + cur.execute( + sql.SQL(GET_MAX_VAL_SIMPLE).format( + schema=sql.Identifier(schema), + table=sql.Identifier(table), + key=sql.Identifier(key), + ), + (start,), + ) + max_val = cur.fetchone()[0] + if max_val is not None: + end = max_val + cur.execute( + sql.SQL(UPDATE_MAX_VAL).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ), + (end, pos), + ) + + # Get actual row count for this block + if is_composite: + where_conditions = [] + pkey_cols = sql.SQL(", ").join(sql.Identifier(col) for col in key_columns) + + where_conditions.append( + sql.SQL("({pkey_cols}) >= ({pkey_values})").format( + pkey_cols=pkey_cols, + pkey_values=sql.SQL(", ").join(sql.Literal(val) for val in start), + ) + ) + + if end is not None: + where_conditions.append( + sql.SQL("({pkey_cols}) <= ({pkey_values})").format( + pkey_cols=pkey_cols, + pkey_values=sql.SQL(", ").join(sql.Literal(val) for val in end), + ) + ) + + where_clause = sql.SQL(" AND ").join(where_conditions) + + cur.execute( + sql.SQL(GET_COUNT_COMPOSITE).format( + schema=sql.Identifier(schema), + table=sql.Identifier(table), + where_clause=where_clause, + ) + ) + else: + cur.execute( + sql.SQL(GET_COUNT_SIMPLE).format( + schema=sql.Identifier(schema), + table=sql.Identifier(table), + key=sql.Identifier(key), + pkey_type=sql.SQL(pkey_type), + ), + (start, end, end), + ) + + count = cur.fetchone()[0] + + # If block is nearly empty, try to merge with either neighbour + if count < target_size * MERGE_THRESHOLD: + # First check if we can merge with previous block + if pos > 0: + if is_composite: + cur.execute( + sql.SQL(GET_BLOCK_COUNT_COMPOSITE).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + schema=sql.Identifier(schema), + table=sql.Identifier(table), + pkey_cols=sql.SQL(", ").join( + sql.Identifier(col) for col in key_columns + ), + ), + (pos - 1,), + ) + else: + cur.execute( + sql.SQL(GET_BLOCK_COUNT_SIMPLE).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + schema=sql.Identifier(schema), + table=sql.Identifier(table), + key=sql.Identifier(key), + ), + (pos - 1,), + ) + + prev_block = cur.fetchone() + if prev_block: + prev_pos, prev_start, prev_end, prev_count = prev_block + if (count + prev_count) <= target_size * 2: + # Move blocks to temp positions + cur.execute( + sql.SQL(UPDATE_NODE_POSITIONS_TEMP).format( + mtree_table=sql.Identifier( + f"ace_mtree_{schema}_{table}" + ), + ), + (TEMP_OFFSET, prev_pos), + ) + + # Merge with previous block - keep full range + cur.execute( + sql.SQL(UPDATE_BLOCK_RANGE_END).format( + mtree_table=sql.Identifier( + f"ace_mtree_{schema}_{table}" + ), + ), + (end, prev_pos), + ) + + # Delete current block at its temporary position + cur.execute( + sql.SQL(DELETE_BLOCK).format( + mtree_table=sql.Identifier( + f"ace_mtree_{schema}_{table}" + ), + ), + (pos + TEMP_OFFSET,), + ) + + # Move remaining blocks back in sequential order + cur.execute( + sql.SQL(UPDATE_NODE_POSITIONS_SEQUENTIAL).format( + mtree_table=sql.Identifier( + f"ace_mtree_{schema}_{table}" + ), + ), + (prev_pos, prev_pos + TEMP_OFFSET), + ) + + modified_positions.add(prev_pos) + + blocks = [ + (p - 1 if p > pos else p, s, e) + for p, s, e in blocks + if p != pos + ] + if not blocks: + break + + if i > 0: + i -= 1 + + continue + + # If we couldn't merge with the previous block, try the next block + if is_composite: + cur.execute( + sql.SQL(GET_BLOCK_COUNT_COMPOSITE).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + schema=sql.Identifier(schema), + table=sql.Identifier(table), + pkey_cols=sql.SQL(", ").join( + sql.Identifier(col) for col in key_columns + ), + ), + (pos + 1,), + ) + else: + cur.execute( + sql.SQL(GET_BLOCK_COUNT_SIMPLE).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + schema=sql.Identifier(schema), + table=sql.Identifier(table), + key=sql.Identifier(key), + ), + (pos + 1,), + ) + + next_block = cur.fetchone() + if next_block: + next_pos, next_start, next_end, next_count = next_block + """ + TODO: handle the case where merging leaves us with a block that + needs splitting. E.g., + - we have, say, 1-1200, 1201-2400, 2401-3600, 3600-6200 + - tablesample is a roll of dice, so this is very much possible + - delete keys 1-4000, + - after merge, we have 1-6200 + - but! 1-6200 is too big. + - in this case, we need to merge and then slate a split for 1-6200 + + I have handled this case, but this needs extensive testing. + """ + if (count + next_count) <= target_size * 2: + # Move blocks to temp positions + cur.execute( + sql.SQL(UPDATE_NODE_POSITIONS_TEMP).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ), + (TEMP_OFFSET, pos), + ) + + # Merge with next block - keep full range + cur.execute( + sql.SQL(UPDATE_BLOCK_RANGE_END).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}") + ), + (next_end, pos), + ) + + # Delete next block at its temporary position + cur.execute( + sql.SQL(DELETE_BLOCK).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}") + ), + (next_pos + TEMP_OFFSET,), + ) + + # Move remaining blocks back in sequential order + cur.execute( + sql.SQL(UPDATE_NODE_POSITIONS_SEQUENTIAL).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}") + ), + (pos, pos + TEMP_OFFSET), + ) + + modified_positions.add(pos) + blocks = [(pos, start, next_end)] + [ + (p - 1 if p > next_pos else p, s, e) + for p, s, e in blocks + if p != next_pos and p != pos + ] + + # check if the current block is empty after merge + if is_composite: + cur.execute( + sql.SQL(GET_BLOCK_COUNT_COMPOSITE).format( + mtree_table=sql.Identifier( + f"ace_mtree_{schema}_{table}" + ), + schema=sql.Identifier(schema), + table=sql.Identifier(table), + pkey_cols=sql.SQL(", ").join( + sql.Identifier(col) for col in key_columns + ), + ), + (pos,), + ) + else: + cur.execute( + sql.SQL(GET_BLOCK_COUNT_SIMPLE).format( + mtree_table=sql.Identifier( + f"ace_mtree_{schema}_{table}" + ), + schema=sql.Identifier(schema), + table=sql.Identifier(table), + key=sql.Identifier(key), + ), + (pos,), + ) + + cur_block = cur.fetchone() + if cur_block: + cur_pos, cur_start, cur_end, cur_count = cur_block + else: + cur_count = 0 + + if not blocks: + break + + if i < len(blocks) and cur_count != 0: + i += 1 + continue + + i += 1 + if i >= len(blocks): + break + + conn.commit() + pbar.close() + return list(modified_positions) + + +def update_mtree(mtree_task: MerkleTreeTask, skip_all_checks=False) -> None: + """ + Update a Merkle tree by recomputing hashes for dirty leaf nodes and new blocks. + Also processes any pending block rebalancing operations. + Uses repeatable read isolation to ensure consistency during the update. + + Args: + cluster_name (str): Name of the cluster + table_name (str): Name of the table to update tree for + dbname (str, optional): Database name. Defaults to None. + """ + + if not skip_all_checks: + ace.merkle_tree_checks(mtree_task, skip_validation=True) + + schema = mtree_task.fields.l_schema + table = mtree_task.fields.l_table + key = mtree_task.fields.key + key_columns = key.split(",") + is_composite = len(key_columns) > 1 + + # we need to first read the metadata to figure out what the + # block size the tree was built with + try: + _, conn = mtree_task.connection_pool.get_cluster_node_connection( + mtree_task.fields.cluster_nodes[0] + ) + cur = conn.cursor() + cur.execute( + sql.SQL(GET_BLOCK_SIZE_FROM_METADATA).format( + schema=sql.Literal(schema), + table=sql.Literal(table), + ) + ) + block_size = cur.fetchone()[0] + except Exception as e: + raise AceException(f"Error getting block size from metadata: {str(e)}") + + if not block_size: + raise AceException(f"Block size not found for {schema}.{table}") + + mtree_task.block_size = block_size + + SPLIT_THRESHOLD = block_size // 2 + # If the number of deletes in a block exceeds 75% of the block size, then we + # merge it + MERGE_THRESHOLD = 0.75 + + for node in mtree_task.fields.cluster_nodes: + + _, conn = mtree_task.connection_pool.connect(node) + conn.set_isolation_level(IsolationLevel.REPEATABLE_READ) + + print(f"\nUpdating Merkle tree on node: {node['name']}") + + if is_composite: + composite_info = CompositeInfo.fetch(conn, f"{schema}_{table}_key_type") + register_composite(composite_info, conn, factory=lambda *args: tuple(args)) + + try: + # Start transaction with repeatable read isolation + with conn.cursor() as cur: + # Get all dirty and new blocks that need updating + cur.execute( + sql.SQL(GET_DIRTY_AND_NEW_BLOCKS).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ) + ) + blocks_to_update = cur.fetchall() + + if not blocks_to_update: + print(f"No updates needed for {node['name']}") + conn.commit() + continue + + # First identify blocks that might need splitting based on insert count + cur.execute( + sql.SQL(FIND_BLOCKS_TO_SPLIT).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}") + ), + [SPLIT_THRESHOLD, [b[0] for b in blocks_to_update]], + ) + blocks_to_split = cur.fetchall() + + if blocks_to_split: + print( + f"Found {len(blocks_to_split)} blocks that may need splitting" + ) + split_blocks( + conn, + schema, + table, + key, + blocks_to_split, + block_size, + ) + + if mtree_task.rebalance: + mtree_table_id = sql.Identifier(f"ace_mtree_{schema}_{table}") + + if is_composite: + cur.execute( + sql.SQL(FIND_BLOCKS_TO_MERGE_COMPOSITE).format( + mtree_table=mtree_table_id, + schema=sql.Identifier(schema), + table=sql.Identifier(table), + key_columns=sql.SQL(", ").join( + [sql.Identifier(col) for col in key_columns] + ), + merge_threshold=MERGE_THRESHOLD, + ), + [ + [b[0] for b in blocks_to_update], + ], + ) + else: + cur.execute( + sql.SQL(FIND_BLOCKS_TO_MERGE_SIMPLE).format( + mtree_table=mtree_table_id, + schema=sql.Identifier(schema), + table=sql.Identifier(table), + key=sql.Identifier(key), + merge_threshold=MERGE_THRESHOLD, + ), + [ + [b[0] for b in blocks_to_update], + ], + ) + blocks_to_merge = cur.fetchall() + + if blocks_to_merge: + print( + f"Found {len(blocks_to_merge)} blocks that may need merging" + ) + merge_blocks( + conn, + schema, + table, + key, + blocks_to_merge, + block_size, + ) + + # Get final list of blocks to update (including newly modified ones) + cur.execute( + sql.SQL(GET_DIRTY_AND_NEW_BLOCKS).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ) + ) + blocks_to_update = cur.fetchall() + + if not blocks_to_update: + print(f"No updates needed for {node['name']}") + conn.commit() + continue + + print(f"Found {len(blocks_to_update)} blocks to update") + + affected_positions = [] + for block in tqdm(blocks_to_update, desc="Recomputing leaf hashes"): + node_position = block[0] + range_start = block[1] + range_end = block[2] + + if is_composite: + where_conditions = [] + + pkey_cols = sql.SQL(", ").join( + sql.Identifier(col) for col in key_columns + ) + where_conditions.append( + sql.SQL("({pkey_cols}) >= ({pkey_values})").format( + pkey_cols=pkey_cols, + pkey_values=sql.SQL(", ").join( + sql.Literal(val) for val in range_start + ), + ) + ) + if range_end is not None: + where_conditions.append( + sql.SQL("({pkey_cols}) <= ({pkey_values})").format( + pkey_cols=pkey_cols, + pkey_values=sql.SQL(", ").join( + sql.Literal(val) for val in range_end + ), + ) + ) + + if not where_conditions: + where_clause = sql.SQL("TRUE") + else: + where_clause = sql.SQL(" AND ").join(where_conditions) + else: + where_conditions = [] + + if range_start is not None: + where_conditions.append( + sql.SQL("{pkey_col} >= {pkey_value}").format( + pkey_col=sql.Identifier(key), + pkey_value=sql.Literal(range_start), + ) + ) + + if range_end is not None: + where_conditions.append( + sql.SQL("{pkey_col} <= {pkey_value}").format( + pkey_col=sql.Identifier(key), + pkey_value=sql.Literal(range_end), + ) + ) + + if not where_conditions: + where_clause = sql.SQL("TRUE") + else: + where_clause = sql.SQL(" AND ").join(where_conditions) + + # Hitting this case would mean that the last block was updated, + # but splitting wasn't necessary. In this case, we need to update + # the range_end to the maximum value of the key. + if range_end is None and range_start is not None: + if is_composite: + pkey_cols = sql.SQL(", ").join( + sql.Identifier(col) for col in key_columns + ) + pkey_values = sql.SQL(", ").join( + sql.Literal(val) for val in range_start + ) + + cur.execute( + sql.SQL(GET_MAX_VAL_COMPOSITE).format( + schema=sql.Identifier(schema), + table=sql.Identifier(table), + pkey_cols=pkey_cols, + pkey_values=pkey_values, + ) + ) + max_vals = cur.fetchone() + if max_vals is not None: + end = max_vals + cur.execute( + sql.SQL(UPDATE_MAX_VAL).format( + mtree_table=sql.Identifier( + f"ace_mtree_{schema}_{table}" + ), + ), + (end, node_position), + ) + else: + cur.execute( + sql.SQL(GET_MAX_VAL_SIMPLE).format( + schema=sql.Identifier(schema), + table=sql.Identifier(table), + key=sql.Identifier(key), + ), + (range_start,), + ) + max_val = cur.fetchone()[0] + if max_val is not None: + end = max_val + cur.execute( + sql.SQL(UPDATE_MAX_VAL).format( + mtree_table=sql.Identifier( + f"ace_mtree_{schema}_{table}" + ), + ), + (end, node_position), + ) + + cur.execute( + sql.SQL(COMPUTE_LEAF_HASHES).format( + schema=sql.Identifier(schema), + table=sql.Identifier(table), + where_clause=where_clause, + columns=sql.SQL(", ").join( + [ + sql.Identifier(col) + for col in mtree_task.fields.cols.split(",") + ] + ), + key=sql.SQL(", ").join( + [sql.Identifier(col) for col in key_columns] + ), + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ) + ) + result = cur.fetchone()[0] + args = { + "leaf_hash": result, + "node_position": node_position, + } + cur.execute( + sql.SQL(UPDATE_LEAF_HASHES).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ), + args, + ) + affected_positions.append(node_position) + + if affected_positions: + cur.execute( + sql.SQL(DELETE_PARENT_NODES).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}") + ) + ) + + level = 0 + + pbar = tqdm( + desc="Building parent nodes", + total=len(affected_positions), + leave=False, + ) + + while True: + cur.execute( + sql.SQL(BUILD_PARENT_NODES).format( + mtree_table=sql.Identifier( + f"ace_mtree_{schema}_{table}" + ), + ), + {"node_level": level}, + ) + + count = cur.fetchone()[0] + if count <= 1: + break + level += 1 + pbar.update(count) + + cur.execute( + sql.SQL(CLEAR_DIRTY_FLAGS).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ), + {"node_positions": affected_positions}, + ) + + conn.commit() + print( + f"Successfully updated {len(affected_positions)} " + "blocks and their parent nodes" + ) + + except Exception as e: + conn.rollback() + traceback.print_exc() + raise AceException( + f"Error updating Merkle tree on {node['name']}: {str(e)}" + ) + + mtree_task.connection_pool.close_all() + mtree_task.scheduler.task_status = "COMPLETED" + mtree_task.scheduler.finished_at = datetime.now() + mtree_task.scheduler.time_taken = util.round_timedelta( + mtree_task.scheduler.finished_at - mtree_task.scheduler.started_at + ).total_seconds() + ace_db.update_ace_task(mtree_task) + + +def find_mismatched_leaves( + conn1, conn2, schema, table, parent_level, parent_position, pbar=None +): + """ + Recursively traverse the merkle tree to find mismatched leaf nodes. + Returns a list of leaf node positions that have different hashes. + """ + mismatched_leaves = set() + + # Get children of this parent node from both nodes + cur1 = conn1.cursor() + cur2 = conn2.cursor() + + params = {"parent_level": parent_level, "parent_position": parent_position} + + cur1.execute( + sql.SQL(GET_NODE_CHILDREN).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ), + params, + ) + node1_children = cur1.fetchall() + + cur2.execute( + sql.SQL(GET_NODE_CHILDREN).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ), + params, + ) + node2_children = cur2.fetchall() + + if pbar is not None: + pbar.update(len(node1_children)) + + # This was a bit tricky to get right: + # 1. It's possible that the there is a mismatch in the tree size between the + # two nides. This could happen if one of the nodes had a flurry of inserts + # or deletes that did not get replicated. + # 2. In such cases, not only do we need to add mismatched leaves to the + # mismatched set, but we need to add all leaves that don't have peers + # on the other nodes. + # + # E.g., consider t1 and t2: + # + # p0 p2 + # /\ / \ + # / \ / \ + # a b / \ + # p1 p0' + # / / \ + # / / \ + # c a' b + # + # -------------- ------------------ + # t1 t2 + # + # Now, say that the leaf node 'a' is different between t1 and t2. While + # recursing down the tree, if at any point, we find a leaf node that does not + # have a peer on the other tree, we need to add it to the mismatched set. + # In the above example, there is no peer for leaf node 'c'. Node 'a' will + # anyway get added since its hashes don't match. + + for child1, child2 in zip_longest(node1_children, node2_children, fillvalue=None): + level1, pos1, hash1 = child1 if child1 else (None, None, None) + level2, pos2, hash2 = child2 if child2 else (None, None, None) + + if hash1 is None or hash2 is None: + if (level1 == 0) or (level2 == 0): + mismatched_leaves.add(pos1 if level1 is not None else pos2) + else: + # Recurse down this branch to find mismatched leaves + child_mismatches = find_mismatched_leaves( + conn1, + conn2, + schema, + table, + level1 if level1 is not None else level2, + pos1 if pos1 is not None else pos2, + pbar, + ) + mismatched_leaves.update(child_mismatches) + elif hash1 != hash2: + if level1 == 0: + mismatched_leaves.add(pos1) + else: + # Recurse down this branch to find mismatched leaves + child1_mismatches = find_mismatched_leaves( + conn1, conn2, schema, table, level1, pos1, pbar + ) + mismatched_leaves.update(child1_mismatches) + + return mismatched_leaves + + +def get_pkey_batches( + node1_conn, node2_conn, schema, table, is_composite, mismatched_positions +): + """ + Get range_start and range_end values for mismatched leaf nodes + and format them into batches for parallel processing. + + This is vastly different from the get_pkey_offsets used in regular table-diff. + Here, we first combine all ranges from both nodes, sort them, and then build + the batches. Here's how it works: + + We will use a problemmatic case that previously led to a significant number of + duplicates to illustrate how this algorithm works. + + Consider the following ranges: + Node 1: [ + ([1], [207]), + ([207], [786]), + ... + ([9676], [10000]), + ([10000], [None]) + ] + + + Node 2 (because it's missing rows 1-8000) [ + ([1], [8079]), + ([8079], [8479]), + ... + ([9676], [10000]), + ([10000], [None]) + ] + + Previously, we would have gotten: + batches = [ + ([1], [8079]), + ([207], [8479]), + ([786], [8809]), + ([1460], [9165]), + ([1988], [9676]), + ([2599], [10000]), + ([3127], [3344]), + ([3344], [3803]), + ([3803], [4610]), + ... + ([10000], [None]) + ] + + Which is clearly wrong, since we have several overlapping ranges that lead to + duplicates in the diff report. + + The correct way is to first combine all ranges, sort them, and then test + the membership of inidividual ranges in the union of all ranges. + + So, after combining and sorting our example ranges, we get: + + batches = [ + ([1], [207]), + ([207], [786]), + ([786], [1460]), + ... + ([9676], [10000]), + ([10000], [None]) + ] + + Testing the membership of computed ranges in the union of all intervals is + important because if both nodes are missing, say, rows 2000-3500, then, even + though we might have computed a range that falls in this interval, it is invalid + because that range does not intersect with any of the [s, e) intervals of the + union of all ranges. + """ + + if is_composite: + node1_composite_info = CompositeInfo.fetch( + node1_conn, f"{schema}_{table}_key_type" + ) + node2_composite_info = CompositeInfo.fetch( + node2_conn, f"{schema}_{table}_key_type" + ) + + register_composite( + node1_composite_info, node1_conn, factory=lambda *args: tuple(args) + ) + register_composite( + node2_composite_info, node2_conn, factory=lambda *args: tuple(args) + ) + + with node1_conn.cursor() as cur1, node2_conn.cursor() as cur2: + cur1.execute( + sql.SQL(GET_LEAF_RANGES).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ), + {"node_positions": mismatched_positions}, + ) + leaf_ranges1 = cur1.fetchall() + + cur2.execute( + sql.SQL(GET_LEAF_RANGES).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ), + {"node_positions": mismatched_positions}, + ) + leaf_ranges2 = cur2.fetchall() + + all_ranges = leaf_ranges1 + leaf_ranges2 + boundaries = [] + for start, end in all_ranges: + if start is not None: + boundaries.append(start) + if end is not None: + boundaries.append(end) + + boundaries = sorted(set(boundaries)) + + slices = [] + for i in range(len(boundaries) - 1): + s = boundaries[i] + e = boundaries[i + 1] + + # We'll form the half-open interval [s, e) + # but only keep it if it intersects any mismatch. + if interval_in_union(s, e, all_ranges): + slices.append((s, e)) + + # TODO: Fix this! + # # We always need the last boundary to be (max_key, None), otherwise, + # # we risk missing diffs. + # last_boundary = boundaries[-1] if boundaries else None + + # if last_boundary is not None: + # slices.append((last_boundary, None)) + + batches = [] + for start, end in slices: + batches.append(([start], [end])) + + return batches + + +def interval_in_union(s, e, intervals): + """ + Returns True if the half-open [s, e) intersects with + the union of all intervals in 'intervals'. + """ + for start, end in intervals: + start = start if start is not None else float("-inf") + end = end if end is not None else float("inf") + + # If [s,e) intersects [start,end) + if not (e <= start or s >= end): + return True + return False + + +def compare_ranges(worker_id, shared_objects, worker_state, work_item): + p_key = shared_objects["p_key"] + schema_name = shared_objects["schema_name"] + table_name = shared_objects["table_name"] + cols = shared_objects["cols_list"] + simple_primary_key = shared_objects["simple_primary_key"] + stop_event = shared_objects["stop_event"] + + node_pair_key, batch = work_item + host1, host2 = node_pair_key.split("/") + + worker_diffs = {} + total_diffs = 0 + + if stop_event.is_set(): + return + + # Process all batches at once if possible + if len(batch) == 1: + pkey1, pkey2 = batch[0] + + where_clause_parts = [] + + if simple_primary_key: + if pkey1[0] is not None: + where_clause_parts.append( + sql.SQL("{p_key} >= {pkey1}").format( + p_key=sql.Identifier(p_key), pkey1=sql.Literal(pkey1[0]) + ) + ) + if pkey2[0] is not None: + where_clause_parts.append( + sql.SQL("{p_key} <= {pkey2}").format( + p_key=sql.Identifier(p_key), pkey2=sql.Literal(pkey2[0]) + ) + ) + else: + if pkey1[0] is not None: + where_clause_parts.append( + sql.SQL("({p_key}) >= ({pkey1})").format( + p_key=sql.SQL(", ").join( + [sql.Identifier(col.strip()) for col in p_key.split(",")] + ), + pkey1=sql.SQL(", ").join( + [sql.Literal(val) for val in pkey1[0]] + ), + ) + ) + if pkey2[0] is not None: + where_clause_parts.append( + sql.SQL("({p_key}) <= ({pkey2})").format( + p_key=sql.SQL(", ").join( + [sql.Identifier(col.strip()) for col in p_key.split(",")] + ), + pkey2=sql.SQL(", ").join( + [sql.Literal(val) for val in pkey2[0]] + ), + ) + ) + + where_clause = ( + sql.SQL(" AND ").join(where_clause_parts) + if where_clause_parts + else sql.SQL("TRUE") + ) + + else: + # Multiple batches - use IN or BETWEEN for better performance + or_clauses = [] + if simple_primary_key: + for pkey1, pkey2 in batch: + and_clauses = [] + + if pkey1 and pkey1[0] is not None: + and_clauses.append( + sql.SQL("{p_key} >= {pkey1}").format( + p_key=sql.Identifier(p_key), pkey1=sql.Literal(pkey1[0]) + ) + ) + if pkey2 and pkey2[0] is not None: + and_clauses.append( + sql.SQL("{p_key} <= {pkey2}").format( + p_key=sql.Identifier(p_key), pkey2=sql.Literal(pkey2[0]) + ) + ) + if and_clauses: + or_clauses.append( + sql.SQL("(") + sql.SQL(" AND ").join(and_clauses) + sql.SQL(")") + ) + else: + # For composite keys, we need to handle each batch separately + # But we can still combine them with OR for a single query + for pkey1, pkey2 in batch: + and_clauses = [] + + if pkey1 and pkey1[0] is not None: + and_clauses.append( + sql.SQL("({p_key}) >= ({pkey1})").format( + p_key=sql.SQL(", ").join( + [ + sql.Identifier(col.strip()) + for col in p_key.split(",") + ] + ), + pkey1=sql.SQL(", ").join( + [sql.Literal(val) for val in pkey1[0]] + ), + ) + ) + + if pkey2 and pkey2[0] is not None: + and_clauses.append( + sql.SQL("({p_key}) <= ({pkey2})").format( + p_key=sql.SQL(", ").join( + [ + sql.Identifier(col.strip()) + for col in p_key.split(",") + ] + ), + pkey2=sql.SQL(", ").join( + [sql.Literal(val) for val in pkey2[0]] + ), + ) + ) + + if and_clauses: + or_clauses.append( + sql.SQL("(") + sql.SQL(" AND ").join(and_clauses) + sql.SQL(")") + ) + + where_clause = ( + sql.SQL(" OR ").join(or_clauses) if or_clauses else sql.SQL("TRUE") + ) + + block_sql = sql.SQL(COMPARE_BLOCKS_SQL).format( + table_name=sql.SQL("{}.{}").format( + sql.Identifier(schema_name), + sql.Identifier(table_name), + ), + where_clause=where_clause, + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [ + executor.submit(ace_core.run_query, worker_state, host1, block_sql), + executor.submit(ace_core.run_query, worker_state, host2, block_sql), + ] + results = [f.result() for f in futures if not f.exception()] + + errors = [f.exception() for f in futures if f.exception()] + + if errors: + return { + "status": BLOCK_ERROR, + "errors": [str(error) for error in errors], + "batch": batch, + "node_pair": node_pair_key, + } + + if len(results) < 2: + return { + "status": BLOCK_ERROR, + "errors": ["Failed to get results from both nodes"], + "batch": batch, + "node_pair": node_pair_key, + } + + t1_result, t2_result = results + + # Use a more efficient approach for comparison + # Create dictionaries keyed by row values for faster lookups + # This avoids the expensive string conversion for every value, and the + # set difference using ordered_set + t1_dict = {} + t2_dict = {} + + for row in t1_result: + row_key = tuple(x.hex() if isinstance(x, bytes) else x for x in row) + t1_dict[row_key] = row + + t2_only = [] + for row in t2_result: + row_key = tuple(x.hex() if isinstance(x, bytes) else x for x in row) + t2_dict[row_key] = row + + if row_key not in t1_dict: + t2_only.append(row_key) + + t1_only = [key for key in t1_dict.keys() if key not in t2_dict] + + # It is possible that the hash mismatch is a false negative. + # E.g., if there are extraneous spaces in the JSONB column. + # In this case, we can still consider the block to be OK. + if not t1_only and not t2_only: + return { + "status": BLOCK_OK, + "diffs": {}, + "total_diffs": 0, + "batch": batch, + "node_pair": node_pair_key, + } + + if node_pair_key not in worker_diffs: + worker_diffs[node_pair_key] = {host1: [], host2: []} + + for row_key in t1_only: + worker_diffs[node_pair_key][host1].append( + dict(zip(cols, (str(x) for x in row_key))) + ) + + for row_key in t2_only: + worker_diffs[node_pair_key][host2].append( + dict(zip(cols, (str(x) for x in row_key))) + ) + + total_diffs = max(len(t1_only), len(t2_only)) + + return { + "status": BLOCK_MISMATCH, + "diffs": worker_diffs, + "total_diffs": total_diffs, + "batch": batch, + "node_pair": node_pair_key, + } + + +def merkle_tree_diff(mtree_task: MerkleTreeTask) -> None: + """ + Compare merkle trees between nodes and perform table diff on mismatched blocks. + """ + ace.merkle_tree_checks(mtree_task, skip_validation=True) + + # It is imperative that we call update_mtree before calling merkle_tree_diff. + # Otherwise, we're comparing stale data. + update_mtree(mtree_task, skip_all_checks=True) + + schema = mtree_task.fields.l_schema + table = mtree_task.fields.l_table + key = mtree_task.fields.key + simple_primary_key = len(key.split(",")) == 1 + + diff_dict = {} + lookup_dict = {} + errors = False + mismatch = False + total_diffs = 0 + diffs_exceeded = False + error_list = [] + + total_rows = 0 + + start_time = datetime.now() + + node_pairs = list(combinations(mtree_task.fields.cluster_nodes, 2)) + + all_node_pair_batches = [] + + for node1, node2 in node_pairs: + print(f"\nComparing merkle trees between {node1['name']} and {node2['name']}") + + _, conn1 = mtree_task.connection_pool.get_cluster_node_connection(node1) + _, conn2 = mtree_task.connection_pool.get_cluster_node_connection(node2) + + try: + cur1 = conn1.cursor() + cur2 = conn2.cursor() + + if not total_rows: + cur1.execute( + sql.SQL(GET_ROW_COUNT_ESTIMATE).format( + schema=sql.Literal(schema), + table=sql.Literal(table), + ) + ) + row_count = cur1.fetchone()[0] + if row_count: + total_rows = row_count * len(mtree_task.fields.node_list) + + cur1.execute( + sql.SQL(GET_ROOT_NODE).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ) + ) + root1 = cur1.fetchone() + + cur2.execute( + sql.SQL(GET_ROOT_NODE).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}"), + ) + ) + root2 = cur2.fetchone() + + if not root1 or not root2: + print(f"No merkle tree found for {schema}.{table}") + continue + + root1_pos, root1_hash = root1 + root2_pos, root2_hash = root2 + + # Get the root level + cur1.execute( + sql.SQL(GET_MAX_NODE_LEVEL).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}") + ) + ) + root_level = cur1.fetchone()[0] + + # If root hashes match, trees are identical + if root1_hash == root2_hash: + print("Merkle trees are identical") + continue + + # Get an estimate of the total number of nodes to traverse + # This is a rough estimate based on the tree structure + # For a binary tree with L levels, the max number of nodes is 2^(L+1) - 1 + # This is not really necessary, but when the nodes are far apart + # (network-wise), or the tree is huge, find_mismatched_leaves() can + # take time. So, we need to show some progress while it's running, + # lest the user should think the script has frozen. + estimated_nodes = 2 ** (root_level + 1) - 1 + + print("Trees differ - traversing to find mismatched leaf nodes...") + + with tqdm(total=estimated_nodes, desc="Traversing merkle tree") as pbar: + mismatched_leaves = list( + find_mismatched_leaves( + conn1, + conn2, + schema, + table, + parent_level=root_level, + parent_position=root1_pos, + pbar=pbar, + ) + ) + pbar.n = estimated_nodes + pbar.refresh() + + if not mismatched_leaves: + print("No mismatched leaf nodes found") + continue + + pkey_batches = get_pkey_batches( + conn1, conn2, schema, table, not simple_primary_key, mismatched_leaves + ) + print(f"Found {len(pkey_batches)} mismatched blocks") + + node_pair_key = f"{node1['name']}/{node2['name']}" + for batch in pkey_batches: + all_node_pair_batches.append((node_pair_key, (node1, node2), batch)) + + except Exception as e: + conn1.rollback() + conn2.rollback() + raise AceException(f"Error comparing merkle trees: {str(e)}") + + finally: + conn1.close() + conn2.close() + + if all_node_pair_batches: + batches_by_pair = {} + for node_pair_key, node_pair, batch in all_node_pair_batches: + if node_pair_key not in batches_by_pair: + batches_by_pair[node_pair_key] = {"node_pair": node_pair, "batches": []} + batches_by_pair[node_pair_key]["batches"].append(batch) + + work_items = [] + for node_pair_key, pair_data in batches_by_pair.items(): + node_pair = pair_data["node_pair"] + + for i in range(0, len(pair_data["batches"]), mtree_task.batch_size): + batch_chunk = pair_data["batches"][i : i + mtree_task.batch_size] + work_items.append((node_pair_key, batch_chunk)) + + print(f"\nProcessing {len(work_items)} batch chunks across all node pairs") + + mtree_task.connection_pool.close_all() + + max_workers = int(os.cpu_count() * mtree_task.max_cpu_ratio) + n_jobs = min(len(work_items), max_workers) + stop_event = Manager().Event() + + shared_objects = { + "p_key": key, + "schema_name": schema, + "table_name": table, + "cols_list": mtree_task.fields.cols.split(","), + "simple_primary_key": simple_primary_key, + "stop_event": stop_event, + "task": mtree_task, + } + + with WorkerPool( + n_jobs=n_jobs, + shared_objects=shared_objects, + use_worker_state=True, + pass_worker_id=True, + ) as pool: + for result in pool.imap_unordered( + compare_ranges, + make_single_arguments(work_items), + iterable_len=len(work_items), + worker_init=ace_core.init_conn_pool, + worker_exit=ace_core.close_conn_pool, + progress_bar=True, + ): + if result["status"] == BLOCK_ERROR: + errors = True + error_list.append( + { + "node_pair": result["node_pair"], + "batch": result["batch"], + "errors": result["errors"], + } + ) + stop_event.set() + break + + if result["status"] == BLOCK_MISMATCH: + mismatch = True + for node_pair, node_diffs in result["diffs"].items(): + if node_pair not in diff_dict: + diff_dict[node_pair] = {} + + if node_pair not in lookup_dict: + lookup_dict[node_pair] = {} + + for node, diffs in node_diffs.items(): + if node not in diff_dict[node_pair]: + diff_dict[node_pair][node] = [] + + if node not in lookup_dict[node_pair]: + lookup_dict[node_pair][node] = set() + + if simple_primary_key: + diff_dict[node_pair][node].extend( + diff + for diff in diffs + if diff[key] not in lookup_dict[node_pair][node] + ) + else: + diff_dict[node_pair][node].extend( + diff + for diff in diffs + if tuple(diff[key] for key in key.split(",")) + not in lookup_dict[node_pair][node] + ) + + if simple_primary_key: + lookup_dict[node_pair][node].update( + diff[key] for diff in diffs + ) + else: + lookup_dict[node_pair][node].update( + tuple(diff[key] for key in key.split(",")) + for diff in diffs + ) + + total_diffs += result["total_diffs"] + if total_diffs >= config.MAX_DIFF_ROWS: + diffs_exceeded = True + stop_event.set() + break + + if diffs_exceeded: + util.message( + "Prematurely terminated jobs since diffs have" + " exceeded MAX_ALLOWED_DIFFS", + p_state="warning", + quiet_mode=mtree_task.quiet_mode, + ) + + run_time = util.round_timedelta(datetime.now() - start_time).total_seconds() + run_time_str = f"{run_time:.2f}" + + mtree_task.scheduler.task_status = "COMPLETED" + mtree_task.scheduler.finished_at = datetime.now() + mtree_task.scheduler.time_taken = run_time + mtree_task.scheduler.task_context = { + "total_rows": total_rows, + "mismatch": mismatch, + "diffs_summary": mtree_task.diff_summary if mismatch else {}, + "errors": [], + } + + print() + + if errors: + context = {"total_rows": total_rows, "mismatch": mismatch, "errors": error_list} + ace.handle_task_exception(mtree_task, context) + + # Even though we've updated the task in the DB, we still need to + # raise an exception so that a) it comes up in the CLI and b) we + # have a record of it in the logs + raise AceException( + "There were one or more errors while running the table-diff job. \n" + "Please examine the connection information provided, or the nodes' \n" + "status before running this script again. Error list: \n" + f"{error_list}" + ) + + # Mismatch is True if there is a block mismatch or if we have + # estimated that diffs may be greater than max allowed diffs + if mismatch: + if diffs_exceeded: + util.message( + f"TABLES DO NOT MATCH. DIFFS HAVE EXCEEDED {config.MAX_DIFF_ROWS} ROWS", + p_state="warning", + quiet_mode=mtree_task.quiet_mode, + ) + + else: + util.message( + "TABLES DO NOT MATCH", + p_state="warning", + quiet_mode=mtree_task.quiet_mode, + ) + + """ + Read the result queue and count differences between each node pair + in the cluster + """ + + for node_pair in diff_dict.keys(): + node1, node2 = node_pair.split("/") + diff_count = max( + len(diff_dict[node_pair][node1]), len(diff_dict[node_pair][node2]) + ) + mtree_task.diff_summary[node_pair] = diff_count + util.message( + f"FOUND {diff_count} DIFFS BETWEEN {node1} AND {node2}", + p_state="warning", + quiet_mode=mtree_task.quiet_mode, + ) + + try: + if mtree_task.output == "json" or mtree_task.output == "html": + mtree_task.diff_file_path = ace.write_diffs_json( + mtree_task, + diff_dict, + mtree_task.fields.col_types, + quiet_mode=mtree_task.quiet_mode, + ) + + if mtree_task.output == "html": + ace_html_reporter.generate_html( + mtree_task.diff_file_path, mtree_task.fields.key.split(",") + ) + elif mtree_task.output == "csv": + ace.write_diffs_csv(diff_dict) + except Exception as e: + context = { + "total_rows": total_rows, + "mismatch": mismatch, + "errors": [str(e)], + } + ace.handle_task_exception(mtree_task, context) + raise e + + else: + util.message( + "TABLES MATCH OK\n", p_state="success", quiet_mode=mtree_task.quiet_mode + ) + + util.message( + f"TOTAL ROWS CHECKED = {total_rows}\nRUN TIME = {run_time_str} seconds", + p_state="info", + quiet_mode=mtree_task.quiet_mode, + ) + + mtree_task.scheduler.task_status = "COMPLETED" + mtree_task.scheduler.finished_at = datetime.now() + mtree_task.scheduler.time_taken = run_time + mtree_task.scheduler.task_context = { + "total_rows": total_rows, + "mismatch": mismatch, + "diffs_summary": mtree_task.diff_summary if mismatch else {}, + "errors": [], + } + + mtree_task.connection_pool.close_all() + ace_db.update_ace_task(mtree_task) + + +def _mtree_init(conn) -> None: + """ + Initialise the database with generic functions needed for Merkle trees. + This only needs to be run once per database. + + Args: + conn: Database connection + """ + cur = conn.cursor() + + # We need pgcrypto for sha256 + cur.execute(sql.SQL("CREATE EXTENSION IF NOT EXISTS pgcrypto;")) + + # We're defining an XOR operator here for building parent hashes + # This is the optimisation Riak uses: + # See: https://www.youtube.com/watch?v=TCiHqF_XTmE + cur.execute(sql.SQL(CREATE_XOR_FUNCTION)) + + # Metadata table where we keep track of tables, their approx. row counts, + # and when the tree was last updated. + cur.execute(sql.SQL("DROP TABLE IF EXISTS ace_mtree_metadata")) + cur.execute(sql.SQL(CREATE_METADATA_TABLE)) + + # Create generic functions for block identification and tracking + cur.execute(sql.SQL(CREATE_BULK_TRIGGER_FUNCTION)) + + conn.commit() + print(f"Merkle tree objects initialised successfully on {conn.info.host}") + + +def mtree_init_helper(mtree_task: MerkleTreeTask) -> None: + """ + CLI helper function to initialise the database with Merkle tree objects + """ + + for node in mtree_task.fields.cluster_nodes: + try: + _, conn = mtree_task.connection_pool.get_cluster_node_connection(node) + _mtree_init(conn) + conn.close() + except Exception as e: + mtree_task.scheduler.task_status = "FAILED" + mtree_task.scheduler.task_context = {"errors": [str(e)]} + raise AceException( + f"Error initialising merkle tree objects on {node}: {str(e)}" + ) + finally: + mtree_task.scheduler.finished_at = datetime.now() + mtree_task.scheduler.time_taken = util.round_timedelta( + mtree_task.scheduler.finished_at - mtree_task.scheduler.started_at + ).total_seconds() + ace_db.update_ace_task(mtree_task) + + mtree_task.scheduler.task_status = "COMPLETED" + mtree_task.scheduler.finished_at = datetime.now() + mtree_task.scheduler.time_taken = util.round_timedelta( + mtree_task.scheduler.finished_at - mtree_task.scheduler.started_at + ).total_seconds() + ace_db.update_ace_task(mtree_task) + + +def _mtree_table_teardown(conn, schema, table) -> None: + """ + Teardown table-specific merkle tree objects + """ + + try: + cur = conn.cursor() + print( + f"Dropping trigger trg_mtree_{schema}_{table}_insert_stmt" + f" on {schema}.{table}" + ) + print( + f"Dropping trigger trg_mtree_{schema}_{table}_update_stmt" + f" on {schema}.{table}" + ) + print( + f"Dropping trigger trg_mtree_{schema}_{table}_delete_stmt" + f" on {schema}.{table}" + ) + cur.execute( + sql.SQL(DROP_MTREE_TRIGGERS).format( + trigger=sql.SQL(f"trg_mtree_{schema}_{table}"), + schema=sql.Identifier(schema), + table=sql.Identifier(table), + ) + ) + + print(f"Dropping table ace_mtree_{schema}_{table}") + + cur.execute( + sql.SQL(DROP_MTREE_TABLE).format( + mtree_table=sql.Identifier(f"ace_mtree_{schema}_{table}") + ) + ) + + conn.commit() + print( + f"Dropped table-specific merkle tree objects on {conn.info.host}" + f" for {schema}.{table}" + ) + + except Exception as e: + raise AceException( + f"Error tearing down table-specific merkle tree objects: {str(e)}" + ) + + +def _mtree_generic_teardown(conn) -> None: + """ + Teardown generic merkle tree objects + """ + + try: + cur = conn.cursor() + print("Dropping XOR function") + cur.execute(sql.SQL(DROP_XOR_FUNCTION)) + + print("Dropping bulk trigger function") + cur.execute(sql.SQL(DROP_BULK_TRIGGER_FUNCTION)) + + print("Dropping metadata table") + cur.execute(sql.SQL(DROP_METADATA_TABLE)) + + conn.commit() + print(f"Dropped generic merkle tree objects on {conn.info.host}") + except Exception as e: + raise AceException(f"Error tearing down generic merkle tree objects: {str(e)}") + + +def _mtree_teardown(conn, schema=None, table=None) -> None: + """ + Core function to teardown generic merkle tree objects and/or table-specific objects + """ + + try: + if schema and table: + _mtree_table_teardown(conn, schema, table) + else: + _mtree_generic_teardown(conn) + + conn.commit() + except Exception as e: + raise AceException(f"Error tearing down merkle tree objects: {str(e)}") + + +def mtree_teardown_helper(mtree_task: MerkleTreeTask) -> None: + """ + Teardown generic merkle tree objects and/or table-specific objects + """ + + for node in mtree_task.fields.cluster_nodes: + try: + _, conn = mtree_task.connection_pool.get_cluster_node_connection(node) + print("=" * 20, f"Node: {node['name']}", "=" * 20) + _mtree_teardown( + conn, + mtree_task.fields.l_schema if mtree_task.fields.l_schema else None, + mtree_task.fields.l_table if mtree_task.fields.l_table else None, + ) + print() + conn.close() + except Exception as e: + mtree_task.scheduler.task_status = "FAILED" + mtree_task.scheduler.task_context = {"errors": [str(e)]} + raise AceException( + f"Error tearing down merkle tree objects on {node['name']}: {str(e)}" + ) + finally: + mtree_task.scheduler.finished_at = datetime.now() + mtree_task.scheduler.time_taken = util.round_timedelta( + mtree_task.scheduler.finished_at - mtree_task.scheduler.started_at + ).total_seconds() + ace_db.update_ace_task(mtree_task) + + mtree_task.scheduler.task_status = "COMPLETED" + mtree_task.scheduler.finished_at = datetime.now() + mtree_task.scheduler.time_taken = util.round_timedelta( + mtree_task.scheduler.finished_at - mtree_task.scheduler.started_at + ).total_seconds() + ace_db.update_ace_task(mtree_task) + + +def check_if_init_needed(conn, schema, funcname): + try: + with conn.cursor() as cur: + cur.execute( + sql.SQL( + """ + SELECT EXISTS ( + SELECT 1 + FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE p.proname = %s + AND n.nspname = %s + ) + """ + ), + (funcname, schema), + ) + res = cur.fetchone() + + if res and res[0]: + # Not a bug; function exists, so we return False + return False + + return True + except Exception as e: + raise AceException(f"Error checking if function exists: {str(e)}") diff --git a/cli/scripts/ace_sql.py b/cli/scripts/ace_sql.py new file mode 100644 index 00000000..d7a96851 --- /dev/null +++ b/cli/scripts/ace_sql.py @@ -0,0 +1,856 @@ +# Various SQL statements for Merkle Tree operations + +CREATE_METADATA_TABLE = """ + CREATE TABLE ace_mtree_metadata ( + schema_name text, + table_name text, + total_rows bigint, + block_size int, + num_blocks int, + is_composite boolean NOT NULL DEFAULT false, + last_updated timestamptz, + PRIMARY KEY (schema_name, table_name) + ) +""" + +# Instead of having one create table and trying to handle cases, I'm just using +# two variations here for simplicity. +CREATE_SIMPLE_MTREE_TABLE = """ + CREATE TABLE {mtree_table} ( + node_level integer NOT NULL, + node_position bigint NOT NULL, + range_start {pkey_type}, + range_end {pkey_type}, + leaf_hash bytea, + node_hash bytea, + dirty boolean DEFAULT false, + inserts_since_tree_update bigint DEFAULT 0, + deletes_since_tree_update bigint DEFAULT 0, + last_modified timestamptz DEFAULT current_timestamp, + PRIMARY KEY (node_level, node_position) + ); + + CREATE INDEX IF NOT EXISTS {range_idx} + ON {mtree_table} (range_start, range_end) + WHERE node_level = 0; +""" + +CREATE_COMPOSITE_MTREE_TABLE = """ + DROP TYPE IF EXISTS {schema}_{table}_key_type CASCADE; + + CREATE TYPE {schema}_{table}_key_type AS ( + {key_type_columns} + ); + + CREATE TABLE {mtree_table} ( + node_level integer NOT NULL, + node_position bigint NOT NULL, + range_start {schema}_{table}_key_type, + range_end {schema}_{table}_key_type, + leaf_hash bytea, + node_hash bytea, + dirty boolean DEFAULT false, + inserts_since_tree_update bigint DEFAULT 0, + deletes_since_tree_update bigint DEFAULT 0, + last_modified timestamptz DEFAULT current_timestamp, + PRIMARY KEY (node_level, node_position) + ); + + CREATE INDEX IF NOT EXISTS {range_idx}_tuple + ON {mtree_table} (range_start, range_end) + WHERE node_level = 0; +""" + +CREATE_GENERIC_TRIGGER = """ + DROP TRIGGER IF EXISTS {trigger}_insert_stmt ON {schema}.{table}; + DROP TRIGGER IF EXISTS {trigger}_update_stmt ON {schema}.{table}; + DROP TRIGGER IF EXISTS {trigger}_delete_stmt ON {schema}.{table}; + + CREATE TRIGGER {trigger}_insert_stmt + AFTER INSERT ON {schema}.{table} + REFERENCING NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE FUNCTION bulk_block_tracking_dispatcher({key}); + + CREATE TRIGGER {trigger}_update_stmt + AFTER UPDATE ON {schema}.{table} + REFERENCING OLD TABLE AS old_table NEW TABLE AS new_table + FOR EACH STATEMENT EXECUTE FUNCTION bulk_block_tracking_dispatcher({key}); + + CREATE TRIGGER {trigger}_delete_stmt + AFTER DELETE ON {schema}.{table} + REFERENCING OLD TABLE AS old_table + FOR EACH STATEMENT EXECUTE FUNCTION bulk_block_tracking_dispatcher({key}); +""" + +# SQL for inserting block ranges for composite keys +INSERT_COMPOSITE_BLOCK_RANGES = """ + INSERT INTO {mtree_table} + (node_level, node_position, range_start, range_end) + VALUES + (0, %s, ROW({start_tuple_values}), ROW({end_tuple_values})); +""" + +ENABLE_ALWAYS = """ + ALTER TABLE {schema}.{table} + ENABLE ALWAYS TRIGGER {insert_trigger}; + ALTER TABLE {schema}.{table} + ENABLE ALWAYS TRIGGER {update_trigger}; + ALTER TABLE {schema}.{table} + ENABLE ALWAYS TRIGGER {delete_trigger}; +""" + +CREATE_XOR_FUNCTION = """ + CREATE OR REPLACE FUNCTION bytea_xor(a bytea, b bytea) RETURNS bytea AS $$ + DECLARE + result bytea; + len int; + BEGIN + IF length(a) != length(b) THEN + RAISE EXCEPTION 'bytea_xor inputs must be same length'; + END IF; + + len := length(a); + result := a; + FOR i IN 0..len-1 LOOP + result := set_byte( + result, i, get_byte(a, i) # get_byte(b, i) + ); + END LOOP; + + RETURN result; + END; + $$ LANGUAGE plpgsql IMMUTABLE STRICT; + + DO $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_operator WHERE oprname = '#' + AND oprleft = 'bytea'::regtype AND oprright = 'bytea'::regtype + ) THEN + CREATE OPERATOR # ( + LEFTARG = bytea, + RIGHTARG = bytea, + PROCEDURE = bytea_xor + ); + END IF; + END $$; +""" + +ESTIMATE_ROW_COUNT = """ + SELECT ( + CASE + WHEN s.n_live_tup > 0 THEN s.n_live_tup + WHEN c.reltuples > 0 THEN c.reltuples + ELSE pg_relation_size(c.oid) / (8192*0.7) + END + )::bigint as estimate + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_stat_user_tables s + ON s.schemaname = n.nspname + AND s.relname = c.relname + WHERE n.nspname = {schema} + AND c.relname = {table} +""" + +GET_PKEY_TYPE = """ + SELECT a.atttypid::regtype::text + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = {schema} + AND c.relname = {table} + AND a.attname = {key} +""" + +UPDATE_METADATA = """ + INSERT INTO ace_mtree_metadata + ( + schema_name, + table_name, + total_rows, + block_size, + num_blocks, + is_composite, + last_updated + ) + VALUES (%s, %s, %s, %s, %s, %s, current_timestamp) + ON CONFLICT (schema_name, table_name) DO UPDATE + SET total_rows = EXCLUDED.total_rows, + block_size = EXCLUDED.block_size, + num_blocks = EXCLUDED.num_blocks, + is_composite = EXCLUDED.is_composite, + last_updated = EXCLUDED.last_updated; +""" + +GET_PKEY_OFFSETS = """ + WITH sampled_data AS ( + SELECT + {key_columns_select} + FROM {schema}.{table} + TABLESAMPLE {table_sample_method}({sample_percent}) + ORDER BY {key_columns_order} + ), + first_row AS ( + SELECT + {key_columns_select} + FROM {schema}.{table} + ORDER BY {key_columns_order} + LIMIT 1 + ), + last_row AS ( + SELECT + {key_columns_select} + FROM {schema}.{table} + ORDER BY {key_columns_order_desc} + LIMIT 1 + ), + sample_boundaries AS ( + SELECT + {key_columns_select}, + ntile({ntile_count}) OVER (ORDER BY {key_columns_order}) as bucket + FROM sampled_data + ), + block_starts AS ( + SELECT DISTINCT ON (bucket) + {key_columns_select} + FROM sample_boundaries + ORDER BY bucket, {key_columns_order} + ), + all_bounds AS ( + SELECT + {first_row_selects}, + 0 as seq + UNION ALL + SELECT + {key_columns_select}, + 1 as seq + FROM block_starts + WHERE ({key_columns_select}) > ( + {first_row_tuple_selects} + ) + UNION ALL + SELECT + {last_row_selects}, + 2 as seq + ), + ranges AS ( + SELECT + {key_columns_select}, + {range_start_columns}, + {range_end_columns}, + seq + FROM all_bounds + ) + SELECT {range_output_columns} + FROM ranges + ORDER BY seq; +""" + +INSERT_BLOCK_RANGES = """ + INSERT INTO {mtree_table} + (node_level, node_position, range_start, range_end, last_modified) + VALUES + (0, %s, %s, %s, current_timestamp); +""" + +COMPUTE_LEAF_HASHES = """ + WITH block_rows AS ( + SELECT * + FROM {schema}.{table} + WHERE {where_clause} + ), + block_hash AS ( + SELECT + digest( + COALESCE( + string_agg( + concat_ws( + '|', + {columns} + ), + '|' + ORDER BY {key} + ), + 'EMPTY_BLOCK' + ), + 'sha256' + ) as leaf_hash + FROM block_rows + ) + SELECT leaf_hash + FROM block_hash; +""" + +UPDATE_LEAF_HASHES = """ + UPDATE {mtree_table} mt + SET + leaf_hash = %(leaf_hash)s, + node_hash = %(leaf_hash)s, + last_modified = current_timestamp + WHERE node_position = %(node_position)s + AND mt.node_level = 0 + RETURNING mt.node_position; +""" + +# COMPUTE_LEAF_HASHES = """ +# WITH block_rows AS ( +# SELECT * +# FROM {schema}.{table} +# WHERE {key} >= %(range_start)s +# AND ({key} < %(range_end)s OR %(range_end)s IS NULL) +# ), +# row_hashes AS ( +# SELECT +# CAST( +# CAST( +# 'x' || MD5( +# {concat_columns} +# ) AS BIT(64) +# ) AS BIGINT +# ) AS row_hash +# FROM block_rows +# ) +# SELECT +# COALESCE( +# CAST( +# BIT_XOR(row_hash) AS VARCHAR +# ), +# 'EMPTY_BLOCK' +# ) AS leaf_hash +# FROM row_hashes; +# """ +# NUMERIC_TO_BYTEA = """ +# CREATE OR REPLACE FUNCTION numeric_to_bytea(n numeric, byte_length int) +# RETURNS bytea AS $$ +# DECLARE +# result bytea := ''; +# i int; +# r int; +# BEGIN +# -- Loop for each byte we want in the result. +# FOR i IN 1..byte_length LOOP +# r := mod(n, 256)::int; +# -- Convert the remainder (one byte) to a 2-digit hex string, +# -- decode it to a bytea, and prepend it. +# result := decode(lpad(to_hex(r), 2, '0'), 'hex') || result; +# n := trunc(n / 256); +# END LOOP; +# RETURN result; +# END; +# $$ LANGUAGE plpgsql IMMUTABLE; +# """ + +# COMPUTE_LEAF_HASHES = """ +# with block_hash as ( +# SELECT numeric_to_bytea(sum(hash_record_extended(t, 0)), 32) as hash +# FROM {schema}.{table} t +# WHERE t.{key} >= %(range_start)s +# AND (t.{key} < %(range_end)s OR %(range_end)s IS NULL) +# ) +# UPDATE ace_mtree_{schema}_{table} +# SET leaf_hash = block_hash.hash, +# node_hash = block_hash.hash, +# last_modified = current_timestamp +# FROM block_hash +# WHERE node_position = %(node_position)s AND node_level = 0 +# RETURNING node_position; +# """ + +GET_BLOCK_RANGES = """ + SELECT node_position, range_start, range_end + FROM {mtree_table} + WHERE node_level = 0 + ORDER BY node_position; +""" + +GET_DIRTY_AND_NEW_BLOCKS = """ + SELECT node_position, range_start, range_end + FROM {mtree_table} + WHERE node_level = 0 + AND (dirty = true OR leaf_hash IS NULL) + ORDER BY node_position; +""" + +CLEAR_DIRTY_FLAGS = """ + UPDATE {mtree_table} + SET dirty = false, + inserts_since_tree_update = 0, + deletes_since_tree_update = 0, + last_modified = current_timestamp + WHERE node_level = 0 + AND node_position = ANY(%(node_positions)s); +""" + +BUILD_PARENT_NODES = """ + WITH pairs AS ( + SELECT + node_level, + node_position / 2 as parent_position, + array_agg(node_hash ORDER BY node_position) as child_hashes + FROM {mtree_table} + WHERE node_level = %(node_level)s + GROUP BY node_level, node_position / 2 + ), + inserted AS ( + INSERT INTO {mtree_table} + (node_level, node_position, node_hash, last_modified) + SELECT + %(node_level)s + 1, + parent_position, + CASE + WHEN array_length(child_hashes, 1) = 1 THEN child_hashes[1] + ELSE child_hashes[1] # child_hashes[2] + END, + current_timestamp + FROM pairs + RETURNING 1 + ) + SELECT count(*) FROM inserted; +""" + + +GET_ROOT_NODE = """ + SELECT node_position, node_hash + FROM {mtree_table} + WHERE node_level = ( + SELECT MAX(node_level) + FROM {mtree_table} + ) +""" + +GET_NODE_CHILDREN = """ + SELECT node_level, node_position, node_hash + FROM {mtree_table} + WHERE node_level = %(parent_level)s - 1 + AND node_position / 2 = %(parent_position)s + ORDER BY node_position +""" + +GET_LEAF_RANGES = """ + SELECT range_start, range_end + FROM {mtree_table} + WHERE node_level = 0 + AND node_position = ANY(%(node_positions)s) + ORDER BY node_position +""" + +GET_ROW_COUNT_ESTIMATE = """ + SELECT total_rows + FROM ace_mtree_metadata + WHERE schema_name = {schema} + AND table_name = {table} +""" + +GET_MAX_VAL_COMPOSITE = """ + SELECT {pkey_cols} + FROM {schema}.{table} + WHERE ({pkey_cols}) >= ({pkey_values}) + ORDER BY ({pkey_cols}) DESC + LIMIT 1 +""" + +UPDATE_MAX_VAL = """ + UPDATE {mtree_table} + SET range_end = %s + WHERE node_level = 0 + AND node_position = %s +""" + +GET_MAX_VAL_SIMPLE = """ + SELECT {key} + FROM {schema}.{table} + WHERE {key} >= %s + ORDER BY {key} DESC + LIMIT 1 +""" + +GET_COUNT_COMPOSITE = """ + SELECT count(*) + FROM {schema}.{table} + WHERE {where_clause} +""" + +GET_COUNT_SIMPLE = """ + SELECT count(*) + FROM {schema}.{table} + WHERE {key} >= %s + AND ({key} < %s OR %s::{pkey_type} IS NULL) +""" + +GET_SPLIT_POINT_COMPOSITE = """ + SELECT ROW({pkey_cols}) + FROM {schema}.{table} + WHERE {where_clause} + ORDER BY {order_cols} + OFFSET %s + LIMIT 1 +""" + +GET_SPLIT_POINT_SIMPLE = """ + SELECT {key} + FROM {schema}.{table} + WHERE {key} >= %s + AND ({key} < %s OR %s::{pkey_type} IS NULL) + ORDER BY {key} + OFFSET %s + LIMIT 1 +""" + +DELETE_PARENT_NODES = """ + DELETE FROM {mtree_table} + WHERE node_level > 0 +""" + +GET_MAX_NODE_POSITION = """ + SELECT MAX(node_position) + 1 + FROM {mtree_table} + WHERE node_level = 0 +""" + +UPDATE_BLOCK_RANGE_END = """ + UPDATE {mtree_table} + SET range_end = %s, + dirty = true, + last_modified = current_timestamp + WHERE node_level = 0 + AND node_position = %s +""" + +UPDATE_NODE_POSITIONS_TEMP = """ + UPDATE {mtree_table} + SET node_position = node_position + %s + WHERE node_level = 0 + AND node_position > %s +""" + +DELETE_BLOCK = """ + DELETE FROM {mtree_table} + WHERE node_level = 0 + AND node_position = %s +""" + +UPDATE_NODE_POSITIONS_SEQUENTIAL = """ + UPDATE {mtree_table} + SET node_position = pos_seq + FROM ( + SELECT node_position, + row_number() OVER ( + ORDER BY node_position + ) + %s as pos_seq + FROM {mtree_table} + WHERE node_level = 0 + AND node_position > %s + ) as seq + WHERE + {mtree_table}.node_position = seq.node_position + AND node_level = 0 +""" + +FIND_BLOCKS_TO_SPLIT = """ + SELECT node_position, range_start, range_end + FROM {mtree_table} + WHERE node_level = 0 + AND inserts_since_tree_update >= %s + AND node_position = ANY(%s) +""" + +FIND_BLOCKS_TO_MERGE_COMPOSITE = """ + WITH range_sizes AS ( + SELECT + mt.node_position, + mt.range_start, + mt.range_end, + mt.deletes_since_tree_update, + COUNT(*) AS current_size + FROM {mtree_table} mt + LEFT JOIN {schema}.{table} t + ON ROW({key_columns}) >= mt.range_start + AND (ROW({key_columns}) < mt.range_end + OR mt.range_end IS NULL) + WHERE mt.node_level = 0 + AND mt.node_position = ANY(%s) + GROUP BY + mt.node_position, + mt.range_start, + mt.range_end, + mt.deletes_since_tree_update + ) + SELECT + node_position, + range_start, + range_end + FROM range_sizes + WHERE + deletes_since_tree_update >= + current_size * {merge_threshold} +""" + +FIND_BLOCKS_TO_MERGE_SIMPLE = """ + WITH range_sizes AS ( + SELECT + mt.node_position, + mt.range_start, + mt.range_end, + mt.deletes_since_tree_update, + COUNT(*) AS current_size + FROM {mtree_table} mt + LEFT JOIN {schema}.{table} t + ON t.{key} >= mt.range_start + AND (t.{key} < mt.range_end + OR mt.range_end IS NULL) + WHERE mt.node_level = 0 + AND mt.node_position = ANY(%s) + GROUP BY + mt.node_position, + mt.range_start, + mt.range_end, + mt.deletes_since_tree_update + ) + SELECT + node_position, + range_start, + range_end + FROM range_sizes + WHERE + deletes_since_tree_update >= + current_size * {merge_threshold} +""" + +GET_BLOCK_COUNT_COMPOSITE = """ + WITH block_data AS + ( + SELECT node_position, range_start, range_end + FROM {mtree_table} + WHERE node_level = 0 + AND node_position = %s + ) + SELECT + b.node_position, + b.range_start, + b.range_end, + COUNT(t.*) AS cnt + FROM block_data b + LEFT JOIN {schema}.{table} t + ON ROW({pkey_cols}) >= b.range_start + AND (ROW({pkey_cols}) <= b.range_end OR b.range_end IS NULL) + GROUP BY + b.node_position, + b.range_start, + b.range_end + ORDER BY b.node_position; +""" + +GET_BLOCK_COUNT_SIMPLE = """ + SELECT node_position, range_start, range_end, count(t.{key}) + FROM {mtree_table} mt + LEFT JOIN {schema}.{table} t + ON t.{key} >= mt.range_start + AND (t.{key} <= mt.range_end OR mt.range_end IS NULL) + WHERE mt.node_level = 0 + AND mt.node_position = %s + GROUP BY mt.node_position, mt.range_start, mt.range_end +""" + +GET_BLOCK_SIZE_FROM_METADATA = """ + SELECT block_size + FROM ace_mtree_metadata + WHERE schema_name = {schema} + AND table_name = {table} +""" + +GET_MAX_NODE_LEVEL = """ + SELECT MAX(node_level) + FROM {mtree_table} +""" + +COMPARE_BLOCKS_SQL = """ + SELECT * FROM {table_name} WHERE {where_clause} +""" + +DROP_XOR_FUNCTION = """ + DROP FUNCTION IF EXISTS bytea_xor(bytea, bytea) CASCADE; +""" + +DROP_METADATA_TABLE = """ + DROP TABLE IF EXISTS ace_mtree_metadata CASCADE; +""" + +DROP_BULK_TRIGGER_FUNCTION = """ + DROP FUNCTION IF EXISTS bulk_block_tracking_dispatcher() CASCADE; +""" + +DROP_MTREE_TABLE = """ + DROP TABLE IF EXISTS {mtree_table} CASCADE; +""" + +DROP_MTREE_TRIGGERS = """ + DROP TRIGGER IF EXISTS {trigger}_insert_stmt ON {schema}.{table}; + DROP TRIGGER IF EXISTS {trigger}_update_stmt ON {schema}.{table}; + DROP TRIGGER IF EXISTS {trigger}_delete_stmt ON {schema}.{table}; +""" + +CREATE_BULK_TRIGGER_FUNCTION = """ +CREATE OR REPLACE FUNCTION bulk_block_tracking_dispatcher() +RETURNS trigger AS $$ +DECLARE + pkey_info text := TG_ARGV[0]; + is_composite boolean; + t_schema text := TG_TABLE_SCHEMA; + t_table text := TG_TABLE_NAME; + mtree_table text := 'ace_mtree_' || t_schema || '_' || t_table; + key_sql text; + max_key_sql text; + max_pos bigint; +BEGIN + SELECT m.is_composite + INTO is_composite + FROM ace_mtree_metadata m + WHERE m.schema_name = t_schema + AND m.table_name = t_table; + + EXECUTE format( + 'SELECT MAX(node_position) FROM %I WHERE node_level = 0', mtree_table + ) INTO max_pos; + + IF is_composite THEN + key_sql := format('ROW(%s)::%I', pkey_info, + t_schema || '_' || t_table || '_key_type'); + ELSE + key_sql := format('%I', pkey_info); + END IF; + + IF TG_OP = 'INSERT' THEN + EXECUTE format($f$ + WITH affected_blocks AS ( + SELECT mt.node_position, count(*) as c + FROM %I mt JOIN new_table nt + ON (%s >= mt.range_start AND + (%s < mt.range_end OR mt.range_end IS NULL OR + (%s <= mt.range_end AND mt.node_position = %L))) + WHERE mt.node_level = 0 GROUP BY mt.node_position + ) + UPDATE %I mt + SET + dirty = true, + inserts_since_tree_update = + mt.inserts_since_tree_update + ab.c, + last_modified = current_timestamp + FROM affected_blocks ab + WHERE mt.node_position = ab.node_position + AND mt.node_level = 0; + $f$, + mtree_table, + replace(key_sql, '%I', 'nt.%I'), + replace(key_sql, '%I', 'nt.%I'), + replace(key_sql, '%I', 'nt.%I'), + max_pos, + mtree_table + ); + + max_key_sql := format( + 'SELECT %s FROM new_table ORDER BY 1 DESC LIMIT 1', key_sql + ); + + ELSIF TG_OP = 'DELETE' THEN + EXECUTE format($f$ + WITH affected_blocks AS ( + SELECT mt.node_position, count(*) as c + FROM %I mt JOIN old_table ot + ON (%s >= mt.range_start AND + (%s < mt.range_end OR mt.range_end IS NULL OR + (%s <= mt.range_end AND mt.node_position = %L))) + WHERE mt.node_level = 0 GROUP BY mt.node_position + ) + UPDATE %I mt + SET + dirty = true, + deletes_since_tree_update = + mt.deletes_since_tree_update + ab.c, + last_modified = current_timestamp + FROM affected_blocks ab + WHERE mt.node_position = ab.node_position + AND mt.node_level = 0; + $f$, + mtree_table, + replace(key_sql, '%I', 'ot.%I'), + replace(key_sql, '%I', 'ot.%I'), + replace(key_sql, '%I', 'ot.%I'), + max_pos, + mtree_table + ); + + ELSIF TG_OP = 'UPDATE' THEN + EXECUTE format($f$ + WITH new_blocks AS ( + SELECT mt.node_position, count(*) as c + FROM %I mt JOIN new_table nt + ON (%s >= mt.range_start AND + (%s < mt.range_end OR mt.range_end IS NULL OR + (%s <= mt.range_end AND mt.node_position = %L))) + WHERE mt.node_level=0 GROUP BY mt.node_position + ), old_blocks AS ( + SELECT mt.node_position + FROM %I mt JOIN old_table ot + ON (%s >= mt.range_start AND + (%s < mt.range_end OR mt.range_end IS NULL OR + (%s <= mt.range_end AND mt.node_position = %L))) + WHERE mt.node_level=0 GROUP BY mt.node_position + ), all_blocks AS ( + SELECT node_position FROM new_blocks + UNION + SELECT node_position FROM old_blocks + ) + UPDATE %I mt + SET + dirty = true, + inserts_since_tree_update = + mt.inserts_since_tree_update + COALESCE(nb.c, 0), + last_modified = current_timestamp + FROM all_blocks ab + LEFT JOIN new_blocks nb ON ab.node_position = nb.node_position + WHERE mt.node_position = ab.node_position + AND mt.node_level=0; + $f$, + mtree_table, + replace(key_sql, '%I', 'nt.%I'), + replace(key_sql, '%I', 'nt.%I'), + replace(key_sql, '%I', 'nt.%I'), + max_pos, + mtree_table, + replace(key_sql, '%I', 'ot.%I'), + replace(key_sql, '%I', 'ot.%I'), + replace(key_sql, '%I', 'ot.%I'), + max_pos, + mtree_table + ); + + max_key_sql := format( + 'SELECT k FROM (SELECT %s k FROM new_table ' || + 'UNION SELECT %s k FROM old_table) q ' || + 'ORDER BY 1 DESC LIMIT 1', + key_sql, key_sql + ); + END IF; + + IF TG_OP != 'DELETE' THEN + EXECUTE format($f$ + WITH affected AS (SELECT (%s) AS max_key) + UPDATE %I + SET + range_end = NULL, + dirty = true, + last_modified = current_timestamp + FROM affected + WHERE node_level = 0 + AND range_end IS NOT NULL + AND affected.max_key > range_end + AND node_position = + (SELECT max(node_position) FROM %I WHERE node_level = 0) + $f$, max_key_sql, mtree_table, mtree_table); + END IF; + + RETURN NULL; +END; +$$ LANGUAGE plpgsql; +""" diff --git a/docs/cli_functions.md b/docs/cli_functions.md index be6bda92..99162704 100644 --- a/docs/cli_functions.md +++ b/docs/cli_functions.md @@ -13,14 +13,25 @@ Use commands in this section to invoke the Active Consistency Engine (ace); comm | Command | Description | |---------|-------------| -| [ace table-diff](functions/ace-table-diff.md) | Compare a table across a cluster and produce a report showing any differences. | -| [ace table-repair](functions/ace-table-repair.md) | Repair a table across a cluster by fixing data inconsistencies identified in a table-diff operation. | -| [ace table-rerun](functions/ace-table-rerun.md) | Reruns a table diff operation based on a previous diff file. | +| [ace mtree](functions/ace-mtree.md) | Use pre-computed table hashes, maintained as Merkle Trees, to achieve a significant speed up over normal-mode table-diff. | | [ace repset-diff](functions/ace-repset-diff.md) | Compare a repset across a cluster and produce a report showing any differences. | | [ace schema-diff](functions/ace-schema-diff.md) | Compare a schema across a cluster and produce a report showing any differences. | | [ace spock-diff](functions/ace-spock-diff.md) | Compare the spock metadata across a cluster and produce a report showing any differences. | -| [ace spock-exception-update](functions/ace-spock-exception-update.md) | Updates the Spock exception status for a specified cluster and node. | +| [ace spock-exception-update](functions/ace-spock-exception-update.md) | Update the Spock exception status for a specified cluster and node. | | [ace start](functions/ace-start.md) | Start the ACE background scheduler and API. | +| [ace table-diff](functions/ace-table-diff.md) | Compare a table across a cluster and produce a report showing differences, if any. | +| [ace table-repair](functions/ace-table-repair.md) | Repair a table across a cluster by fixing data inconsistencies identified in a table-diff operation. | +| [ace table-rerun](functions/ace-table-rerun.md) | Rerun a table diff operation based on a previous diff file. | + +### ace mtree submodule commands + +| Command | Description | +|---------|-------------| +| [ace mtree](functions/ace-mtree-build.md) | Builds a new Merkle tree for a table. | +| [ace mtree](functions/ace-mtree-init.md) | Initialises the database with necessary objects for Merkle trees. | +| [ace mtree](functions/ace-mtree-table-diff.md) | Compares Merkle trees of a table across cluster nodes. | +| [ace mtree](functions/ace-mtree-teardown.md) | Removes Merkle tree objects. | +| [ace mtree](functions/ace-mtree-update.md) | Updates an existing Merkle tree. | ## cluster module commands Use commands in the cluster module to create and modify a cluster; commands in the cluster module include: diff --git a/docs/functions/ace-mtree-build.md b/docs/functions/ace-mtree-build.md new file mode 100644 index 00000000..e6868767 --- /dev/null +++ b/docs/functions/ace-mtree-build.md @@ -0,0 +1,44 @@ + +## SYNOPSIS + ./pgedge ace mtree build CLUSTER_NAME TABLE_NAME + +## DESCRIPTION + Builds a new Merkle tree for a table. + +## POSITIONAL ARGUMENTS + CLUSTER_NAME + Name of the cluster. + TABLE_NAME + Schema-qualified table name. + +## FLAGS + -d, --dbname=DBNAME + Name of the database. + + -a, --analyse=ANALYSE + Run ANALYZE on the table. + + --recreate_objects=RECREATE_OBJECTS + Drop and recreate Merkle tree objects. + + -b, --block_size=BLOCK_SIZE + Rows per leaf block. + + -m, --max_cpu_ratio=MAX_CPU_RATIO + Max CPU for parallel operations. + + -w, --write_ranges=WRITE_RANGES + Write block ranges to a JSON file. + + --ranges_file=RANGES_FILE + Path to a file with pre-computed ranges. + + -n, --nodes=NODES + Comma-separated subset of nodes. + + -q, --quiet_mode=QUIET_MODE + Suppress output. + + -s, --skip_block_size_check=SKIP_BLOCK_SIZE_CHECK + Skip block size check, and potentially tolerate unsafe block sizes. Defaults to False. + diff --git a/docs/functions/ace-mtree-init.md b/docs/functions/ace-mtree-init.md new file mode 100644 index 00000000..139bd2af --- /dev/null +++ b/docs/functions/ace-mtree-init.md @@ -0,0 +1,21 @@ + +## SYNOPSIS + ./pgedge ace mtree init CLUSTER_NAME + +## DESCRIPTION + Initialises the database with necessary objects for Merkle trees. + +## POSITIONAL ARGUMENTS + CLUSTER_NAME + Name of the cluster. + +## FLAGS + -d, --dbname=DBNAME + Name of the database. + + -n, --nodes=NODES + Comma-separated subset of nodes. + + -q, --quiet_mode=QUIET_MODE + Suppress output. + diff --git a/docs/functions/ace-mtree-table-diff.md b/docs/functions/ace-mtree-table-diff.md new file mode 100644 index 00000000..e1e00d49 --- /dev/null +++ b/docs/functions/ace-mtree-table-diff.md @@ -0,0 +1,35 @@ + +## SYNOPSIS + ./pgedge ace mtree table-diff CLUSTER_NAME TABLE_NAME + +## DESCRIPTION + Compares Merkle trees of a table across cluster nodes. + +## POSITIONAL ARGUMENTS + CLUSTER_NAME + Name of the cluster. + TABLE_NAME + Schema-qualified table name. + +## FLAGS + -d, --dbname=DBNAME + Name of the database. + + -r, --rebalance=REBALANCE + Trigger rebalancing of the tree. + + -m, --max_cpu_ratio=MAX_CPU_RATIO + Max CPU for parallel operations. + + -b, --batch_size=BATCH_SIZE + Number of blocks per worker batch. + + -n, --nodes=NODES + Comma-separated subset of nodes. + + -o, --output=OUTPUT + Output format (json, csv, html). + + -q, --quiet_mode=QUIET_MODE + Suppress output. + diff --git a/docs/functions/ace-mtree-teardown.md b/docs/functions/ace-mtree-teardown.md new file mode 100644 index 00000000..c621b94e --- /dev/null +++ b/docs/functions/ace-mtree-teardown.md @@ -0,0 +1,24 @@ + +## SYNOPSIS + ./pgedge ace mtree teardown CLUSTER_NAME + +## DESCRIPTION + Removes Merkle tree objects. + +## POSITIONAL ARGUMENTS + CLUSTER_NAME + Name of the cluster. + +## FLAGS + -t, --table_name=TABLE_NAME + Schema-qualified table name. If omitted, removes objects for the entire database. + + -d, --dbname=DBNAME + Name of the database. + + -n, --nodes=NODES + Comma-separated subset of nodes. + + -q, --quiet_mode=QUIET_MODE + Suppress output. + diff --git a/docs/functions/ace-mtree-update.md b/docs/functions/ace-mtree-update.md new file mode 100644 index 00000000..6e284658 --- /dev/null +++ b/docs/functions/ace-mtree-update.md @@ -0,0 +1,29 @@ + +## SYNOPSIS + ./pgedge ace mtree update CLUSTER_NAME TABLE_NAME + +## DESCRIPTION + Updates an existing Merkle tree. + +## POSITIONAL ARGUMENTS + CLUSTER_NAME + Name of the cluster. + TABLE_NAME + Schema-qualified table name. + +## FLAGS + -d, --dbname=DBNAME + Name of the database. + + -r, --rebalance=REBALANCE + Trigger rebalancing of the tree. + + -m, --max_cpu_ratio=MAX_CPU_RATIO + Max CPU for parallel operations. + + -n, --nodes=NODES + Comma-separated subset of nodes. + + -q, --quiet_mode=QUIET_MODE + Suppress output. + diff --git a/docs/functions/ace-mtree.md b/docs/functions/ace-mtree.md new file mode 100644 index 00000000..4aa98431 --- /dev/null +++ b/docs/functions/ace-mtree.md @@ -0,0 +1,14 @@ + +## SYNOPSIS + ./pgedge ace mtree COMMAND + +## DESCRIPTION + Use pre-computed table hashes, maintained as Merkle Trees, to achieve a significant speed up over normal-mode table-diff. + +## COMMANDS + COMMAND is one of the following: + build # Builds a new Merkle tree for a table. + init # Initialises the database with necessary objects for Merkle trees. + table-diff # Compares Merkle trees of a table across cluster nodes. + teardown # Removes Merkle tree objects. + update # Updates an existing Merkle tree. diff --git a/docs/functions/ace-repset-diff.md b/docs/functions/ace-repset-diff.md index 43d903fb..c2f3a560 100644 --- a/docs/functions/ace-repset-diff.md +++ b/docs/functions/ace-repset-diff.md @@ -13,10 +13,10 @@ ## FLAGS -d, --dbname=DBNAME - Name of the database. Defaults to the name of the first database in the cluster configuration. + Name of the database to use. If omitted, defaults to the first database in the cluster configuration. - --block_rows=BLOCK_ROWS - Number of rows to process per block. Defaults to config.BLOCK_ROWS_DEFAULT. + --block_size=BLOCK_SIZE + Number of rows to process per block. Defaults to config.DIFF_BLOCK_SIZE. -m, --max_cpu_ratio=MAX_CPU_RATIO Maximum CPU utilisation. The accepted range is 0.0-1.0. Defaults to config.MAX_CPU_RATIO_DEFAULT. @@ -25,17 +25,17 @@ Output format. Acceptable values are "json", "csv", and "html". Defaults to "json". -n, --nodes=NODES - Comma-delimited subset of nodes on which the command will be executed. Defaults to "all". + Comma-separated subset of nodes on which the command will be executed. Defaults to "all". --batch_size=BATCH_SIZE - Size of each batch. Defaults to config.BATCH_SIZE_DEFAULT. + Size of each batch, i.e., number of blocks each worker should process. Defaults to config.DIFF_BATCH_SIZE. -q, --quiet=QUIET Whether to suppress output in stdout. Defaults to False. --skip_tables=SKIP_TABLES - Comma-deliminated list of tables to skip. + Comma-separated list of tables to skip. If omitted, no tables are skipped. --skip_file=SKIP_FILE - Path to a file containing a list of tables to skip. + Path to a file containing a list of tables to skip. If omitted, no tables are skipped. diff --git a/docs/functions/ace-spock-exception-update.md b/docs/functions/ace-spock-exception-update.md index d4dc90fb..b7838c2f 100644 --- a/docs/functions/ace-spock-exception-update.md +++ b/docs/functions/ace-spock-exception-update.md @@ -3,7 +3,7 @@ ./pgedge ace spock-exception-update CLUSTER_NAME NODE_NAME ENTRY ## DESCRIPTION - Updates the Spock exception status for a specified cluster and node. + Update the Spock exception status for a specified cluster and node. ## POSITIONAL ARGUMENTS CLUSTER_NAME @@ -11,7 +11,19 @@ NODE_NAME The name of the node within the cluster where the update should be performed. ENTRY - A JSON string representing the exception entry. The JSON object + A JSON string representing the exception entry. Should contain the following keys. + + - "remote_origin" (str) transaction that caused the exception. (Required) + + - "remote_commit_ts" (str) transaction on the remote origin. (Required) + + - "remote_xid" (str) (Required) + + - "status" (str) "RESOLVED", "IGNORED"). (Required) + + - "resolution_details" (dict, optional) dictionary containing details about the resolution. + + - "command_counter" (int, optional) exception detail (matching this command_counter along with remote_origin, remote_commit_ts, remote_xid) in the `spock.exception_status_detail` table is updated. If omitted, the main entry in `spock.exception_status` and all related detail entries for the (remote_origin, remote_commit_ts, remote_xid) trio in `spock.exception_status_detail` are updated. ## FLAGS -d, --dbname=DBNAME diff --git a/docs/functions/ace-table-diff.md b/docs/functions/ace-table-diff.md index 463b5103..791975f9 100644 --- a/docs/functions/ace-table-diff.md +++ b/docs/functions/ace-table-diff.md @@ -3,7 +3,7 @@ ./pgedge ace table-diff CLUSTER_NAME TABLE_NAME ## DESCRIPTION - Compare a table across a cluster and produce a report showing any differences. + Compare a table across a cluster and produce a report showing differences, if any. ## POSITIONAL ARGUMENTS CLUSTER_NAME @@ -13,10 +13,10 @@ ## FLAGS -d, --dbname=DBNAME - Name of the database. Defaults to the name of the first database in the cluster configuration. + Name of the database to use. If omitted, defaults to the first database in the cluster configuration file. --block_rows=BLOCK_ROWS - Number of rows to process per block. Defaults to config.BLOCK_ROWS_DEFAULT. + Number of rows to process per block. Defaults to config.DIFF_BLOCK_SIZE. -m, --max_cpu_ratio=MAX_CPU_RATIO Maximum CPU utilisation. The accepted range is 0.0-1.0. Defaults to config.MAX_CPU_RATIO_DEFAULT. @@ -25,14 +25,18 @@ Output format. Acceptable values are "json", "csv", and "html". Defaults to "json". -n, --nodes=NODES - Comma-delimited subset of nodes on which the command will be executed. Defaults to "all". + Comma-separated subset of nodes on which the command will be executed. Defaults to "all". --batch_size=BATCH_SIZE - Size of each batch. Defaults to config.BATCH_SIZE_DEFAULT. + Size of each batch, i.e., number of blocks each worker should process. Defaults to config.DIFF_BATCH_SIZE. -t, --table_filter=TABLE_FILTER - A SQL WHERE clause that allows you to filter rows for comparison. + Used to compare a subset of rows in the table. Specified as a WHERE clause of a SQL query. E.g., --table-filter="customer_id < 100" will compare only rows with customer_id less than 100. If omitted, the entire table is compared. -q, --quiet=QUIET Whether to suppress output in stdout. Defaults to False. + -s, --skip_block_size_check=SKIP_BLOCK_SIZE_CHECK + Skip block size check, and potentially tolerate unsafe block sizes. Defaults to False. + + Additional flags are accepted. diff --git a/docs/functions/ace-table-repair.md b/docs/functions/ace-table-repair.md index 23bf3e09..43d86ad6 100644 --- a/docs/functions/ace-table-repair.md +++ b/docs/functions/ace-table-repair.md @@ -39,7 +39,7 @@ If True, fixes null values in the table columns by looking at the corresponding column in the other nodes. Does not need the source of truth to be specified. Must be used only in special cases. This is not a recommended option for repairing divergence. Defaults to False. --fire_triggers=FIRE_TRIGGERS - If True, instructs triggers to fire when a repair is performed; note that ENABLE ALWAYS triggers will fire regardless of the value. + If True, fires triggers on a table, if any, during the repair process. Note that ENABLE ALWAYS triggers will fire regardless of the value. -b, --bidirectional=BIDIRECTIONAL If True, performs a bidirectional repair, applies differences found between nodes to create a distinct union of the content. In a distinct union, each row that is missing is recreated on the node from which it is missing, eventually leading to a data set (on all nodes) in which all rows are represented exactly once. diff --git a/docs/functions/ace-table-rerun.md b/docs/functions/ace-table-rerun.md index 9fe01def..70e50e54 100644 --- a/docs/functions/ace-table-rerun.md +++ b/docs/functions/ace-table-rerun.md @@ -3,7 +3,7 @@ ./pgedge ace table-rerun CLUSTER_NAME DIFF_FILE TABLE_NAME ## DESCRIPTION - Reruns a table diff operation based on a previous diff file. + Rerun a table diff operation based on a previous diff file. ## POSITIONAL ARGUMENTS CLUSTER_NAME @@ -15,11 +15,11 @@ ## FLAGS -d, --dbname=DBNAME - Name of the database. Defaults to the name of the first database in the cluster configuration. + Name of the database to use. If omitted, defaults to the first database in the cluster configuration. + + -b, --behavior=BEHAVIOR + Deprecated. Formerly used to specify the behavior of the rerun. Now, it always defaults to "hostdb". -q, --quiet=QUIET Whether to suppress output in stdout. Defaults to False. - -b, --behavior=BEHAVIOR - The rerun behavior, either "multiprocessing" or "hostdb". "multiprocessing" uses parallel processing for faster execution. "hostdb" uses the host database to create temporary tables for faster comparisons. Defaults to "multiprocessing". - diff --git a/docs/functions/ace.md b/docs/functions/ace.md index 3926085b..4deb1e57 100644 --- a/docs/functions/ace.md +++ b/docs/functions/ace.md @@ -2,13 +2,17 @@ ## SYNOPSIS ./pgedge ace COMMAND +## DESCRIPTION + The Active Consistency Engine of pgEdge. + ## COMMANDS COMMAND is one of the following: - table-diff # Compare a table across a cluster and produce a report showing any differences. - table-repair # Repair a table across a cluster by fixing data inconsistencies identified in a table-diff operation. - table-rerun # Reruns a table diff operation based on a previous diff file. + mtree # Use pre-computed table hashes, maintained as Merkle Trees, to achieve a significant speed up over normal-mode table-diff. repset-diff # Compare a repset across a cluster and produce a report showing any differences. schema-diff # Compare a schema across a cluster and produce a report showing any differences. spock-diff # Compare the spock metadata across a cluster and produce a report showing any differences. - spock-exception-update# Updates the Spock exception status for a specified cluster and node. + spock-exception-update# Update the Spock exception status for a specified cluster and node. start # Start the ACE background scheduler and API. + table-diff # Compare a table across a cluster and produce a report showing differences, if any. + table-repair # Repair a table across a cluster by fixing data inconsistencies identified in a table-diff operation. + table-rerun # Rerun a table diff operation based on a previous diff file. diff --git a/requirements.txt b/requirements.txt index 08407427..604ab605 100644 --- a/requirements.txt +++ b/requirements.txt @@ -37,5 +37,5 @@ urllib3==2.2.2 PyYAML==6.0.1 python-etcd==0.4.5 python-dateutil==2.9.0 -cryptography==44.0.0 +cryptography==44.0.1 ydiff==1.3 From 4d23c3cb170e345921bcdfa297ce1c4dde27c0b0 Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Thu, 26 Jun 2025 11:49:39 -0500 Subject: [PATCH 15/42] bump version to 25.1.0 (#338) --- cli/scripts/install.py | 2 +- cli/scripts/util.py | 2 +- env.sh | 4 ++-- src/conf/versions.sql | 3 ++- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cli/scripts/install.py b/cli/scripts/install.py index c8094ee4..31225eba 100644 --- a/cli/scripts/install.py +++ b/cli/scripts/install.py @@ -3,7 +3,7 @@ import sys, os, tarfile, platform -VER = "25.0.0" +VER = "25.1.0" REPO = os.getenv("REPO", "https://pgedge-download.s3.amazonaws.com/REPO") if sys.version_info < (3, 9): diff --git a/cli/scripts/util.py b/cli/scripts/util.py index fa98b265..b4335a21 100644 --- a/cli/scripts/util.py +++ b/cli/scripts/util.py @@ -4,7 +4,7 @@ import os import time -MY_VERSION = "25.0.0" +MY_VERSION = "25.1.0" MY_CODENAME = "" DEFAULT_PG = "16" diff --git a/env.sh b/env.sh index 0acb87eb..f65ed386 100755 --- a/env.sh +++ b/env.sh @@ -1,5 +1,5 @@ -hubV=25.0.0 -hubVV=25.0.0 +hubV=25.1.0 +hubVV=25.1.0 aceV=$hubV kirkV=$hubV diff --git a/src/conf/versions.sql b/src/conf/versions.sql index 1b220420..36a51883 100644 --- a/src/conf/versions.sql +++ b/src/conf/versions.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS hub; CREATE TABLE hub(v TEXT NOT NULL PRIMARY KEY, c TEXT NOT NULL, d TEXT NOT NULL); -INSERT INTO hub VALUES ('25.0.0', 'Constellation', '20250603'); +INSERT INTO hub VALUES ('25.1.0', '', '20250626'); DROP VIEW IF EXISTS v_versions; DROP VIEW IF EXISTS v_products; @@ -139,6 +139,7 @@ INSERT INTO projects VALUES ('hub', 'app', 0, 0, 'hub', 0, 'https://github.com/p INSERT INTO releases VALUES ('hub', 1, 'hub', '', '', 'hidden', '', 1, '', '', ''); INSERT INTO versions VALUES ('hub', (select v from hub), '', 1, (select d from hub), '', '', ''); +INSERT INTO versions VALUES ('hub', '25.0.0', '', 0, '20250603', '', '', ''); INSERT INTO versions VALUES ('hub', '24.10.13', '', 0, '20250509', '', '', ''); INSERT INTO versions VALUES ('hub', '24.10.11', '', 0, '20250224', '', '', ''); INSERT INTO versions VALUES ('hub', '24.10.10', '', 0, '20250123', '', '', ''); From 898073fd9838b1873f7f64acddc3ad3cf8130e9e Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Mon, 30 Jun 2025 15:38:39 +0500 Subject: [PATCH 16/42] Spock 5 integration --- cli/scripts/cluster.py | 5 +- cli/scripts/meta.py | 1 - cli/scripts/setup.py | 3 +- cli/scripts/util.py | 120 ++++++++++++++++++++++++++++++++++++++++- src/conf/versions.sql | 22 ++------ 5 files changed, 127 insertions(+), 24 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index a4915c4a..8f3bf6db 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -6,6 +6,7 @@ import meta import time import sys +import setup_core import getpass from tabulate import tabulate # type: ignore from ipaddress import ip_address @@ -1312,7 +1313,9 @@ def init(cluster_name, install=True): parsed_json = get_cluster_json(cluster_name) if parsed_json is None: util.exit_message("Unable to load cluster JSON", 1) - + pg_version = db_settings["pg_version"] + spock_ver = db_settings.get("spock_version", util.get_default_spock(pg_version)) + util.validate_spock_pg_compat(spock_ver, pg_version) verbose = parsed_json.get("log_level", "info") all_nodes = nodes.copy() diff --git a/cli/scripts/meta.py b/cli/scripts/meta.py index b399e43a..9cd5fb1c 100644 --- a/cli/scripts/meta.py +++ b/cli/scripts/meta.py @@ -306,7 +306,6 @@ def get_default_spock(pgv): + pgv + "' \n" + " AND component LIKE 'spock%'" - + " AND version not LIKE '%devel%'" ) try: c = con.cursor() diff --git a/cli/scripts/setup.py b/cli/scripts/setup.py index 2fc21aa9..6e5452d8 100755 --- a/cli/scripts/setup.py +++ b/cli/scripts/setup.py @@ -95,7 +95,8 @@ def setup_pgedge(User=None, Passwd=None, dbName=None, port=None, pg_data=None, p pg_ver = df_pg pg_major, pg_minor = setup_core.parse_pg(pg_ver) - + # Validate that if Spock ≥5 we're using PG 15.13+, 16.9+ or 17.5+Add commentMore actions + util.validate_spock_pg_compat(spock_ver, pg_ver) pg_init_options = "" if pg_data is not None: pg_data = pg_data.rstrip("/") diff --git a/cli/scripts/util.py b/cli/scripts/util.py index b4335a21..4a0ef6ea 100644 --- a/cli/scripts/util.py +++ b/cli/scripts/util.py @@ -8,8 +8,8 @@ MY_CODENAME = "" DEFAULT_PG = "16" -DEFAULT_SPOCK = "40" -DEFAULT_SPOCK_17 = "40" +DEFAULT_SPOCK = "50" +DEFAULT_SPOCK_17 = "50" MY_CMD = os.getenv("MY_CMD", None) MY_HOME = os.getenv("MY_HOME", None) MY_LIBS = f"{MY_HOME}/hub/scripts/lib" @@ -37,6 +37,7 @@ import subprocess import getpass import filecmp +import re from subprocess import Popen, PIPE, STDOUT from datetime import datetime, timedelta from urllib import request as urllib2 @@ -123,7 +124,122 @@ def get_default_spock(pgv): return(DEFAULT_SPOCK) +def get_default_spock(pgv): + if pgv == "17": + return(DEFAULT_SPOCK_17) + + return(DEFAULT_SPOCK) + + + +def validate_spock_pg_compat(spock_ver: str = None, pg_ver: str = None) -> None: + """ + Compatibility rules: + • If Spock < 5.0.0 ⇒ works with any supported PostgreSQL major. + • If Spock ≥ 5.0.0 ⇒ + – PG15 must be ≥ 15.13 + – PG16 must be ≥ 16.9 + – PG17 must be ≥ 17.5 + + Also supports shorthand Spock strings: + – "50" → "5.0.0", "40" → "4.0.0", etc. + """ + # 0) Fill in defaults if user didn’t pass anything + if not pg_ver: + pg_ver = DEFAULT_PG + if not spock_ver: + maj = int(pg_ver.split(".", 1)[0]) + spock_ver = DEFAULT_SPOCK_17 if maj == 17 else DEFAULT_SPOCK + + # 0.5) Normalize two-digit shorthand (e.g. "50" → "5.0.0") + m = re.fullmatch(r'(\d)(\d)$', spock_ver) + if m: + spock_ver = f"{int(m.group(1))}.{int(m.group(2))}.0" + + # 1) Parse Spock version (abort on bad format) + try: + spv = Version(spock_ver) + except ValueError: + exit_message(f"Invalid Spock version '{spock_ver}'. Aborting.", 1, isJSON) + + # 2) If Spock < 5 ⇒ compatible with any PG + if spv.major < 5: + return + # — New block: handle pg_ver with “-1” or “-2” suffix + rev = None + rev_match = re.fullmatch(r'(\d+)\.(\d+)-(1|2)$', pg_ver) + if rev_match: + pg_major = int(rev_match.group(1)) + pg_patch = int(rev_match.group(2)) + rev = int(rev_match.group(3)) + + # reject revision “-1” on Spock ≥5 + if rev == 1: + exit_message( + f"Error: PostgreSQL {pg_major}.{pg_patch}-1 is not supported with Spock {spv}; " + "please use the “-2” revision instead.", + 1, + isJSON + ) + # for “-2”, we strip suffix and proceed with pg_major/pg_patch below + # end new block + + # 3) Spock ≥ 5 ⇒ enforce minimum‐patch for each PG major + minimum_patches = { + 15: 13, + 16: 9, + 17: 5, + } + + # 4) Extract PG major and patch (if not already set by rev_match) + if rev_match: + # pg_major, pg_patch are already set + pass + elif "." not in pg_ver: + # bare-major → use its minimum patch + try: + pg_major = int(pg_ver) + except ValueError: + exit_message(f"Invalid PostgreSQL version '{pg_ver}'. Aborting.", 1, isJSON) + if pg_major not in minimum_patches: + allowed = ", ".join(str(m) for m in minimum_patches) + exit_message( + f"Error: Spock {spv} supports only PostgreSQL majors {allowed}; " + f"you have {pg_major}. Aborting.", + 1, + isJSON + ) + pg_patch = minimum_patches[pg_major] + else: + parts = pg_ver.split(".", 2) + if len(parts) < 2: + exit_message(f"Invalid PostgreSQL version '{pg_ver}'. Aborting.", 1, isJSON) + try: + pg_major = int(parts[0]) + pg_patch = int(parts[1]) + except ValueError: + exit_message(f"Invalid PostgreSQL version '{pg_ver}'. Aborting.", 1, isJSON) + + # 5) Major must be supported + if pg_major not in minimum_patches: + allowed = ", ".join(str(m) for m in minimum_patches) + exit_message( + f"Error: Spock {spv} supports only PostgreSQL majors {allowed}; " + f"you have {pg_major}. Aborting.", + 1, + isJSON + ) + + # 6) Enforce minimum‐patch + required = minimum_patches[pg_major] + if pg_patch < required: + exit_message( + f"Error: Spock {spv} requires PostgreSQL {pg_major}.{required} or newer; " + f"you have {pg_major}.{pg_patch}. Aborting.", + 1, + isJSON + ) def get_cpu_info(): try: import cpuinfo diff --git a/src/conf/versions.sql b/src/conf/versions.sql index 36a51883..2e09f6d5 100644 --- a/src/conf/versions.sql +++ b/src/conf/versions.sql @@ -74,11 +74,6 @@ CREATE TABLE extensions ( preload_name TEXT NOT NULL, default_conf TEXT NOT NULL ); -INSERT INTO extensions VALUES ('spock33', 'spock', 1, 'spock', - 'wal_level=logical | max_worker_processes=12 | max_replication_slots=16 | - max_wal_senders=16 | hot_standby_feedback=on | wal_sender_timeout=5s | - track_commit_timestamp=on | spock.conflict_resolution=last_update_wins | - spock.save_resolutions=on | spock.conflict_log_level=DEBUG'); INSERT INTO extensions VALUES ('spock40', 'spock', 1, 'spock', 'wal_level=logical | max_worker_processes=12 | max_replication_slots=16 | max_wal_senders=16 | hot_standby_feedback=on | wal_sender_timeout=5s | @@ -320,17 +315,6 @@ INSERT INTO versions VALUES ('snowflake-pg17', '2.2-1', 'amd, arm', 1, '20240626 -- ## SPOCK (parent project) ############ INSERT INTO projects VALUES ('spock', 'pge', 4, 0, '', 1, 'https://github.com/pgedge/spock/tags', 'spock', 1, 'spock.png', 'Logical Rep w/ Conflict Resolution', 'https://github.com/pgedge/spock/', 'pg_spock, pgsspock, vulcan'); - --- ## SPOCK33 ########################### -INSERT INTO releases VALUES ('spock33-pg15', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); -INSERT INTO releases VALUES ('spock33-pg16', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); - -INSERT INTO versions VALUES ('spock33-pg15', '3.3.6-1', 'amd, arm', 1, '20240820', 'pg15', '', ''); -INSERT INTO versions VALUES ('spock33-pg16', '3.3.6-1', 'amd, arm', 1, '20240820', 'pg16', '', ''); - -INSERT INTO versions VALUES ('spock33-pg15', '3.3.5-1', 'amd, arm', 0, '20240607', 'pg15', '', ''); -INSERT INTO versions VALUES ('spock33-pg16', '3.3.5-1', 'amd, arm', 0, '20240607', 'pg16', '', ''); - -- ## SPOCK40 ########################### INSERT INTO releases VALUES ('spock40-pg15', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); INSERT INTO releases VALUES ('spock40-pg16', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); @@ -350,9 +334,9 @@ INSERT INTO versions VALUES ('spock40-pg17', '4.0.8-1', 'amd, arm', 0, '20241218 -- ## spock50 ########################### -INSERT INTO releases VALUES ('spock50-pg15', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); -INSERT INTO releases VALUES ('spock50-pg16', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); -INSERT INTO releases VALUES ('spock50-pg17', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); +INSERT INTO releases VALUES ('spock50-pg15', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); +INSERT INTO releases VALUES ('spock50-pg16', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); +INSERT INTO releases VALUES ('spock50-pg17', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); INSERT INTO versions VALUES ('spock50-pg15', '5.0.0-devel1-1', 'amd, arm', 1, '20250521', 'pg15', '', ''); INSERT INTO versions VALUES ('spock50-pg16', '5.0.0-devel1-1', 'amd, arm', 1, '20250521', 'pg16', '', ''); From 32d523351974aba0d8e0eb82e45c7934bc202731 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Mon, 30 Jun 2025 20:21:03 +0500 Subject: [PATCH 17/42] function duplication removed from util.py --- cli/scripts/util.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cli/scripts/util.py b/cli/scripts/util.py index 4a0ef6ea..22f1e5f4 100644 --- a/cli/scripts/util.py +++ b/cli/scripts/util.py @@ -124,12 +124,6 @@ def get_default_spock(pgv): return(DEFAULT_SPOCK) -def get_default_spock(pgv): - if pgv == "17": - return(DEFAULT_SPOCK_17) - - return(DEFAULT_SPOCK) - def validate_spock_pg_compat(spock_ver: str = None, pg_ver: str = None) -> None: From 7c518ddb9c59d61c0fd324f1473cc0f162e5eaeb Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Mon, 30 Jun 2025 16:37:30 -0500 Subject: [PATCH 18/42] support --rm-data if custom data directory is in use (#340) --- src/pgXX/remove-pgXX.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/pgXX/remove-pgXX.py b/src/pgXX/remove-pgXX.py index 28203cff..fcd77ce9 100644 --- a/src/pgXX/remove-pgXX.py +++ b/src/pgXX/remove-pgXX.py @@ -15,5 +15,6 @@ isRM_DATA = os.getenv("isRM_DATA", "False") if isRM_DATA == "True": util.message("Removing 'data' directories at your request") - util.echo_cmd(f"sudo rm -r data/{pgver}") + data_dir = util.get_column("datadir", pgver) + util.echo_cmd(f"sudo rm -r {data_dir}") util.echo_cmd(f"sudo rm -r data/logs/{pgver}") From 6d12a4e6b259d65f678ce484c063577e3a22a97c Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Tue, 1 Jul 2025 20:42:51 +0500 Subject: [PATCH 19/42] Pg17 is default --- cli/scripts/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/scripts/util.py b/cli/scripts/util.py index 22f1e5f4..273f1134 100644 --- a/cli/scripts/util.py +++ b/cli/scripts/util.py @@ -7,7 +7,7 @@ MY_VERSION = "25.1.0" MY_CODENAME = "" -DEFAULT_PG = "16" +DEFAULT_PG = "17" DEFAULT_SPOCK = "50" DEFAULT_SPOCK_17 = "50" MY_CMD = os.getenv("MY_CMD", None) From 4a54bc631548948669c94d9f2ded53d74c686679 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Tue, 1 Jul 2025 20:43:14 +0500 Subject: [PATCH 20/42] build_all.sh default pg17 --- build_all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_all.sh b/build_all.sh index f584fbdf..3f8f5f22 100755 --- a/build_all.sh +++ b/build_all.sh @@ -13,7 +13,7 @@ if [ ! $num_p == "0" ] && [ ! $num_p == "1" ]; then fi if [ "$1" == "" ]; then - majorV=16 + majorV=17 echo "" echo "### Defaulting to pg $majorV ###" else From 02c13d0984ba9ec179e5f44e8e91a82945aee2fc Mon Sep 17 00:00:00 2001 From: Gabrielle Poncey Date: Tue, 1 Jul 2025 11:06:36 -0700 Subject: [PATCH 21/42] fix: db guc set supports quoted params\n Adds an extra set of quotations to the guc_value portion of the ALTER SYSTEM SET SQL in order to support spaced, quoted guc values \n PLAT-44 --- cli/scripts/db.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cli/scripts/db.py b/cli/scripts/db.py index f5138bcc..3174a389 100755 --- a/cli/scripts/db.py +++ b/cli/scripts/db.py @@ -109,18 +109,20 @@ def create(db=None, User=None, Passwd=None, pg=None, spock=None, help=False): def guc_set(guc_name, guc_value): """Set GUC.""" - pg_v, spock_v = util.get_pg_v() pg = pg_v[2:] nc = "./pgedge " ncb = nc + "pgbin " + str(pg) + " " - cmd = f"ALTER SYSTEM SET {guc_name} = {guc_value}" + cmd = f"ALTER SYSTEM SET {guc_name} = '{guc_value}'" + rc1 = util.echo_cmd(ncb + '"psql -q -c \\"' + cmd + '\\" postgres"',False) cmd = f"SELECT pg_reload_conf()" rc2 = util.echo_cmd(ncb + '"psql -q -c \\"' + cmd + '\\" postgres"',False) + rcs = rc1 + rc2 + if rcs == 0: util.message(f"Set GUC {guc_name} to {guc_value}","info") else: From 8db8b9a5a60a57430708834143ca7bc85657377c Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Wed, 2 Jul 2025 18:57:07 +0500 Subject: [PATCH 22/42] spock5 issue with regression test --- build_all.sh | 2 +- cli/scripts/util.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/build_all.sh b/build_all.sh index f584fbdf..3f8f5f22 100755 --- a/build_all.sh +++ b/build_all.sh @@ -13,7 +13,7 @@ if [ ! $num_p == "0" ] && [ ! $num_p == "1" ]; then fi if [ "$1" == "" ]; then - majorV=16 + majorV=17 echo "" echo "### Defaulting to pg $majorV ###" else diff --git a/cli/scripts/util.py b/cli/scripts/util.py index 22f1e5f4..a8dc0ab2 100644 --- a/cli/scripts/util.py +++ b/cli/scripts/util.py @@ -7,7 +7,7 @@ MY_VERSION = "25.1.0" MY_CODENAME = "" -DEFAULT_PG = "16" +DEFAULT_PG = "17" DEFAULT_SPOCK = "50" DEFAULT_SPOCK_17 = "50" MY_CMD = os.getenv("MY_CMD", None) @@ -138,12 +138,18 @@ def validate_spock_pg_compat(spock_ver: str = None, pg_ver: str = None) -> None: Also supports shorthand Spock strings: – "50" → "5.0.0", "40" → "4.0.0", etc. """ - # 0) Fill in defaults if user didn’t pass anything + # 0) Fill in defaults if user didn’t pass anything if not pg_ver: pg_ver = DEFAULT_PG + else: + pg_ver = str(pg_ver) # ← force to string + if not spock_ver: maj = int(pg_ver.split(".", 1)[0]) spock_ver = DEFAULT_SPOCK_17 if maj == 17 else DEFAULT_SPOCK + else: + spock_ver = str(spock_ver) # ← force to string + # 0.5) Normalize two-digit shorthand (e.g. "50" → "5.0.0") m = re.fullmatch(r'(\d)(\d)$', spock_ver) From 584e593f2630b85f0e71133ccbcffa0c0515a30a Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Wed, 2 Jul 2025 23:13:39 +0500 Subject: [PATCH 23/42] pg1-host issue resolved --- src/backrest/backrest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/backrest/backrest.py b/src/backrest/backrest.py index 2a805509..997742fe 100755 --- a/src/backrest/backrest.py +++ b/src/backrest/backrest.py @@ -30,8 +30,7 @@ def fetch_config(): "repo1-retention-full-type", "repo1-path", "repo1-host-user", "repo1-host", "repo1-cipher-type", "log-level-console", "repo1-type", "process-max", "compress-level", "pg1-path", - "pg1-user", "pg1-database", "db-socket-path", "pg1-port", - "pg1-host" + "pg1-user", "pg1-database", "db-socket-path", "pg1-port" ] for param in params: config[param] = util.get_value("BACKUP", param) From c7c492e91f1a93d025d24c4c978f24a9bf57f0ea Mon Sep 17 00:00:00 2001 From: Gabrielle Poncey Date: Thu, 3 Jul 2025 10:00:27 -0700 Subject: [PATCH 24/42] fix: db guc set supports quoted params Adds an extra set of quotations to the guc_value and escapes / or quotations it may hold to allow for spaced and quoted parameters to be passed PLAT-44 --- cli/scripts/db.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/scripts/db.py b/cli/scripts/db.py index 3174a389..ae1e57d2 100755 --- a/cli/scripts/db.py +++ b/cli/scripts/db.py @@ -8,6 +8,7 @@ import json import util import fire +import re def create(db=None, User=None, Passwd=None, pg=None, spock=None, help=False): """ @@ -109,12 +110,14 @@ def create(db=None, User=None, Passwd=None, pg=None, spock=None, help=False): def guc_set(guc_name, guc_value): """Set GUC.""" + pg_v, spock_v = util.get_pg_v() pg = pg_v[2:] nc = "./pgedge " ncb = nc + "pgbin " + str(pg) + " " + guc_value = re.sub(r'([\'\\])', r'\1\1', str(guc_value)) cmd = f"ALTER SYSTEM SET {guc_name} = '{guc_value}'" rc1 = util.echo_cmd(ncb + '"psql -q -c \\"' + cmd + '\\" postgres"',False) From 844ea41420fb3f8a58585c0911390725e3e92987 Mon Sep 17 00:00:00 2001 From: Gabrielle Poncey Date: Thu, 3 Jul 2025 11:35:05 -0700 Subject: [PATCH 25/42] json-create supports hostnames node config json-create allows creation of a node or subnode with either ip address or hostname through the use of socket.gethostbyname PLAT-131 --- cli/scripts/cluster.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index d6a1fa42..8fccada3 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -8,10 +8,11 @@ import sys import getpass from tabulate import tabulate # type: ignore -from ipaddress import ip_address +import socket import os import re + BASE_DIR = "cluster" DEFAULT_REPO = "https://pgedge-download.s3.amazonaws.com/REPO" @@ -1042,9 +1043,11 @@ def get_cluster_info(cluster_name): ) try: if public_ip: - ip_address(public_ip) + socket.gethostbyname(public_ip) + if private_ip: - ip_address(private_ip) + socket.gethostbyname(private_ip) + except ValueError: validation_errors.append( f"Invalid IP address provided for node {node.get('name')}." @@ -1077,9 +1080,9 @@ def get_cluster_info(cluster_name): ) try: if public_ip: - ip_address(public_ip) + socket.gethostbyname(public_ip) if private_ip: - ip_address(private_ip) + socket.gethostbyname(private_ip) except ValueError: validation_errors.append( f"Invalid IP address provided for sub-node {sub_node.get('name')}." From 87a36a5284cce9cbbade77ee3dadb77bd4676140 Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Tue, 8 Jul 2025 16:24:22 +0500 Subject: [PATCH 26/42] pg1-host remove from install-backrest module --- src/backrest/install-backrest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/backrest/install-backrest.py b/src/backrest/install-backrest.py index 461168d1..e9ffef32 100644 --- a/src/backrest/install-backrest.py +++ b/src/backrest/install-backrest.py @@ -58,7 +58,6 @@ def configure_backup_settings(): "pg1-path": "xx", "pg1-user": "xx", "pg1-port": "5432", - "pg1-host": "127.0.0.1", "db-socket-path": "/tmp", "global:archive-push": { "compress-level": "3" From cd0a520e8fc0d7836ef1dc4f86cb08ada68bda3c Mon Sep 17 00:00:00 2001 From: Gabrielle Poncey Date: Tue, 8 Jul 2025 10:00:55 -0700 Subject: [PATCH 27/42] fix: appropriate error caught invalid host/ip In the case that a hostname or ip address cannot be resolved by gethostbyname, socket error will be caught, appended to validation_errors and reported with the specific node from which it arose from. PLAT-131 --- cli/scripts/cluster.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index 8fccada3..d90619c3 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -1048,9 +1048,9 @@ def get_cluster_info(cluster_name): if private_ip: socket.gethostbyname(private_ip) - except ValueError: + except socket.gaierror as e: validation_errors.append( - f"Invalid IP address provided for node {node.get('name')}." + f"Error resolving hostname or ip adress for node {node.get('name')} : {e}." ) for sub_node in node.get("sub_nodes", []): @@ -1083,9 +1083,9 @@ def get_cluster_info(cluster_name): socket.gethostbyname(public_ip) if private_ip: socket.gethostbyname(private_ip) - except ValueError: + except socket.gaierror as e: validation_errors.append( - f"Invalid IP address provided for sub-node {sub_node.get('name')}." + f"Error resolving hostname or ip adress for sub-node {sub_node.get('name')}: {e}." ) if validation_errors: From f20c9baacf1952ff120b63049ee2e352f9c65ff1 Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Tue, 15 Jul 2025 13:40:57 +0500 Subject: [PATCH 28/42] [SPOC-84]: Updates spock versions --- env.sh | 4 ++-- src/conf/versions.sql | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/env.sh b/env.sh index f65ed386..07a0eb0a 100755 --- a/env.sh +++ b/env.sh @@ -8,7 +8,7 @@ bundle=pgedge api=pgedge ctlibsV=1.6 -spock50V=5.0.0-devel1-1 +spock50V=5.0.0-1 spock40V=4.0.10-1 @@ -16,7 +16,7 @@ spock33V=3.3.6-1 # removeComponentFromOut: Specifies the Spock component (e.g., spock50) to exclude from stable mode builds in make_tgz.sh. # This variable is ignored in current mode builds, which include all components. -removeComponentFromOut=spock50 +removeComponentFromOut= lolorV=1.2-1 snwflkV=2.2-1 diff --git a/src/conf/versions.sql b/src/conf/versions.sql index 2e09f6d5..f10e9127 100644 --- a/src/conf/versions.sql +++ b/src/conf/versions.sql @@ -152,17 +152,17 @@ INSERT INTO projects VALUES ('pg', 'pge', 1, 5432, '', 1, 'https://github.com/po INSERT INTO releases VALUES ('pg15', 2, 'pg', '', '', 'prod', 'New in 2022', 1, 'POSTGRES', '', ''); -INSERT INTO versions VALUES ('pg15', '15.13-2', 'amd, arm', 1, '20250619','', '', ''); +INSERT INTO versions VALUES ('pg15', '15.13-2', 'amd, arm', 1, '20250715','', '', ''); INSERT INTO versions VALUES ('pg15', '15.12-1', 'amd, arm', 0, '20250224','', '', ''); INSERT INTO releases VALUES ('pg16', 2, 'pg', '', '', 'prod', 'New in 2023!', 1, 'POSTGRES', '', ''); -INSERT INTO versions VALUES ('pg16', '16.9-2', 'amd, arm', 1, '20250619','', '', ''); +INSERT INTO versions VALUES ('pg16', '16.9-2', 'amd, arm', 1, '20250715','', '', ''); INSERT INTO versions VALUES ('pg16', '16.8-1', 'amd, arm', 0, '20250224','', '', ''); INSERT INTO releases VALUES ('pg17', 2, 'pg', '', '', 'prod', 'New in 2024!', 1, 'POSTGRES', '', ''); -INSERT INTO versions VALUES ('pg17', '17.5-2', 'amd, arm', 1, '20250619','', '', ''); +INSERT INTO versions VALUES ('pg17', '17.5-2', 'amd, arm', 1, '20250715','', '', ''); INSERT INTO versions VALUES ('pg17', '17.4-1', 'amd, arm', 0, '20250224','', '', ''); -- ## ORAFCE ############################# @@ -338,9 +338,9 @@ INSERT INTO releases VALUES ('spock50-pg15', 4, 'spock', 'Spock', '', 'prod', '' INSERT INTO releases VALUES ('spock50-pg16', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); INSERT INTO releases VALUES ('spock50-pg17', 4, 'spock', 'Spock', '', 'prod', '', 1, 'pgEdge Community', '', ''); -INSERT INTO versions VALUES ('spock50-pg15', '5.0.0-devel1-1', 'amd, arm', 1, '20250521', 'pg15', '', ''); -INSERT INTO versions VALUES ('spock50-pg16', '5.0.0-devel1-1', 'amd, arm', 1, '20250521', 'pg16', '', ''); -INSERT INTO versions VALUES ('spock50-pg17', '5.0.0-devel1-1', 'amd, arm', 1, '20250521', 'pg17', '', ''); +INSERT INTO versions VALUES ('spock50-pg15', '5.0.0-1', 'amd, arm', 1, '20250715', 'pg15', '', ''); +INSERT INTO versions VALUES ('spock50-pg16', '5.0.0-1', 'amd, arm', 1, '20250715', 'pg16', '', ''); +INSERT INTO versions VALUES ('spock50-pg17', '5.0.0-1', 'amd, arm', 1, '20250715', 'pg17', '', ''); -- ## LOLOR ############################# INSERT INTO projects VALUES ('lolor', 'pge', 4, 0, '', 1, 'https://github.com/pgedge/lolor/tags', From 5da69caf71bcf57a3e2e61f74dc963002b3790f2 Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Tue, 15 Jul 2025 17:09:38 +0500 Subject: [PATCH 29/42] [BR-86]: Updates script to copy artifacts in devel subdirectory (#351) * [BR-86]: Updates copy-to-devel.sh script so that it takes two more inputs i.e. mode (stable/current) and the s3 subdirectory within stable/current --- devel/util/copy-to-devel.sh | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/devel/util/copy-to-devel.sh b/devel/util/copy-to-devel.sh index a2f0ed78..8ab38d15 100755 --- a/devel/util/copy-to-devel.sh +++ b/devel/util/copy-to-devel.sh @@ -21,13 +21,34 @@ if [ ! -d $outDir ]; then exit 1 fi +# Check if $2 is empty +if [[ -z "$2" ]]; then + echo "Error: Second argument (mode) is required and must be either 'stable' or 'current'." + exit 1 +fi + +# Validate allowed values +if [[ "$2" != "stable" && "$2" != "current" ]]; then + echo "Error: Second argument must be either 'stable' or 'current'." + exit 1 +fi + +# Check if $3 is empty +if [[ -z "$3" ]]; then + echo "Error: Third argument (subdir) is required and it is a subdirectory inside 'stable' or 'current'." + exit 1 +fi + +MODE=$2 +SUBDIR=$3 + sleep 2 cd $outDir ls sleep 2 flags="--acl public-read --storage-class STANDARD --recursive" -BR=$BUCKET/REPO +BR=$BUCKET/REPO/$MODE/$SUBDIR set -x aws --region $REGION s3 cp . $BR $flags @@ -38,7 +59,7 @@ sleep 2 # content disposition header if [ $rc -eq 0 ] && [ -f "$offline_tgz_bndl" ]; then echo "Uploading offline bundle with content-disposition header" - aws --region $REGION s3 cp "$offline_tgz_bndl" "$BUCKET/REPO/" \ + aws --region $REGION s3 cp "$offline_tgz_bndl" "$BUCKET/REPO/$MODE/$SUBDIR/" \ --acl public-read \ --content-disposition "attachment; filename=$offline_tgz_bndl" rc=$? # Capture exit code from second upload From e00e69b1c5d9c5cc80454c5d56d08e2587f254bd Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Mon, 21 Jul 2025 08:42:02 -0500 Subject: [PATCH 30/42] spock module command fixes (#354) * resolve error with spock set-readonly * remove spock node-alter-location * add missing dependency --- cli/scripts/db.py | 1 + cli/scripts/spock.py | 33 +-------------------- docs/cli_functions.md | 1 - docs/functions/spock-node-alter-location.md | 11 ------- docs/functions/spock.md | 1 - 5 files changed, 2 insertions(+), 45 deletions(-) delete mode 100644 docs/functions/spock-node-alter-location.md diff --git a/cli/scripts/db.py b/cli/scripts/db.py index ae1e57d2..9f9bf0f5 100755 --- a/cli/scripts/db.py +++ b/cli/scripts/db.py @@ -9,6 +9,7 @@ import util import fire import re +import psycopg def create(db=None, User=None, Passwd=None, pg=None, spock=None, help=False): """ diff --git a/cli/scripts/spock.py b/cli/scripts/spock.py index 3376b66c..3bec2756 100644 --- a/cli/scripts/spock.py +++ b/cli/scripts/spock.py @@ -198,36 +198,6 @@ def node_drop(node_name, db): sys.exit(0) -def node_alter_location(node_name, location, db): - """Set location details for spock node.""" - pg_v,spock_v = get_spock_ver() - - [location_nm, country, state, lattitude, longitude] = util.get_location_dtls( - location - ) - - sql = """ -UPDATE spock.node - SET location_nm = ?, country = ?, state = ?, lattitude = ?, longitude = ? - WHERE location = ? -""" - - con = util.get_pg_connection(pg_v, db, util.get_user()) - - rc = 0 - try: - con = util.get_pg_connection(pg_v, "postgres", util.get_user()) - cur = con.cursor(row_factory=psycopg.rows.dict_row) - cur.execute(sql, [location_nm, country, state, lattitude, longitude]) - con.commit() - except Exception as e: - util.print_exception(e) - con.rollback() - rc = 1 - - sys.exit(rc) - - def node_list(db): """Display node table. @@ -726,7 +696,7 @@ def set_readonly(readonly="off"): util.message("spock.set_readonly() deprecated, use db.set_readonly() instead", "warning") - return(db.set_readonly(readonly, pg)) + return(db.set_readonly(readonly)) def get_pii_cols(db, schema=None): @@ -1017,7 +987,6 @@ def metrics_check(db): { "node-create": node_create, "node-drop": node_drop, - "node-alter-location": node_alter_location, "node-list": node_list, "node-add-interface": node_add_interface, "node-drop-interface": node_drop_interface, diff --git a/docs/cli_functions.md b/docs/cli_functions.md index 99162704..f7f5abdc 100644 --- a/docs/cli_functions.md +++ b/docs/cli_functions.md @@ -96,7 +96,6 @@ Use commands in this section to invoke spock extension functionality with the CL |---------|-------------| | [spock node-create](functions/spock-node-create.md) | Define a node for spock. | | [spock node-drop](functions/spock-node-drop.md) | Remove a spock node. | -| [spock node-alter-location](functions/spock-node-alter-location.md) | Set location details for spock node. | | [spock node-list](functions/spock-node-list.md) | Display node table. | | [spock node-add-interface](functions/spock-node-add-interface.md) | Add a new node interface. | | [spock node-drop-interface](functions/spock-node-drop-interface.md) | Delete a node interface. | diff --git a/docs/functions/spock-node-alter-location.md b/docs/functions/spock-node-alter-location.md deleted file mode 100644 index 9d39297b..00000000 --- a/docs/functions/spock-node-alter-location.md +++ /dev/null @@ -1,11 +0,0 @@ - -## SYNOPSIS - ./pgedge spock node-alter-location NODE_NAME LOCATION DB - -## DESCRIPTION - Set location details for spock node. - -## POSITIONAL ARGUMENTS - NODE_NAME - LOCATION - DB diff --git a/docs/functions/spock.md b/docs/functions/spock.md index d1f26fdc..5ed3198c 100644 --- a/docs/functions/spock.md +++ b/docs/functions/spock.md @@ -6,7 +6,6 @@ COMMAND is one of the following: node-create # Define a node for spock. node-drop # Remove a spock node. - node-alter-location # Set location details for spock node. node-list # Display node table. node-add-interface # Add a new node interface. node-drop-interface # Delete a node interface. From f8436d6340204b11d9b9ca71d1a32ceaa2dadb94 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Thu, 24 Jul 2025 01:24:48 +0500 Subject: [PATCH 31/42] add validation to um install for spock5 upgrade (#356) * major spock version upgrade command (draft) * successfully upgraded to spock5 using um upgrade-spock * spock 5 upgrade support * Update um.py Upgrade-spock function hide * code refactored * additional test case added for spock 5 upgrade comp check * additional test case added for spock 5 upgrade comp check * additional test case added for spock 5 upgrade comp check * warning banner removed from case 3 * final adjustment to logging output --------- Co-authored-by: Matthew Mols --- cli/scripts/um.py | 90 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 85 insertions(+), 5 deletions(-) diff --git a/cli/scripts/um.py b/cli/scripts/um.py index f07aedbc..ceff8e3a 100755 --- a/cli/scripts/um.py +++ b/cli/scripts/um.py @@ -3,8 +3,9 @@ import os, sys, glob, sqlite3, time import fire, meta, util - isJSON = util.isJSON +import re +import sqlite3 as _sqlite3 MY_HOME = util.MY_HOME @@ -83,20 +84,29 @@ def update(): run_cmd("update") + +# Match both "spock50" and "spock50-pg17" (and variants like spock5, spock5-pg16) +_SPOCK5_NAME_RE = re.compile(r"^spock5(?:0)?(?:-pg\d+)?$", re.IGNORECASE) +_SPOCK5_VER_RE = re.compile(r"^5\.", re.IGNORECASE) +_SPOCK50_RE = re.compile(r"^spock50(?:-pg\d+)?$", re.IGNORECASE) + def install(component, active=True): """Install a component.""" + # Trigger pre-check ONLY for Spock 5.0 artifacts + if _SPOCK50_RE.match(component): + validate_spock_upgrade() # should sys.exit(1) on failure; otherwise just returns + + # Common path (no duplication) if active not in (True, False): util.exit_message("'active' parm must be True or False") - - cmd = "install" - if active is False: - cmd = "install --no-preload" + cmd = "install" if active else "install --no-preload" util.message(f"um.install({cmd} {component})", "debug") run_cmd(cmd, component) + def remove(component): """Uninstall a component.""" installed_comp_list = meta.get_component_list() @@ -234,6 +244,76 @@ def verify_metadata(Project="", Stage="prod", IsCurrent=0): meta.pretty_sql(sql) +def validate_spock_upgrade(): + """ + Validate Spock↔PostgreSQL compatibility for an upcoming Spock 5 install. + """ + + DB_PATH = "data/conf/db_local.db" + + SPOCK_SQL = """ + SELECT version AS spock_ver, component AS spock_comp + FROM components + WHERE component LIKE 'spock%' + ORDER BY CASE + WHEN version LIKE '5.%' THEN 5 + WHEN version LIKE '4.%' THEN 4 + ELSE 0 + END DESC, + version DESC + LIMIT 1; + """ + + PG_SQL = """ + SELECT version AS pg_ver + FROM components + WHERE component LIKE 'pg__' + ORDER BY CAST(substr(component, 3) AS INTEGER) DESC + LIMIT 1; + """ + + try: + with _sqlite3.connect(DB_PATH) as conn: + sp_row = conn.execute(SPOCK_SQL).fetchone() + pg_row = conn.execute(PG_SQL).fetchone() + except _sqlite3.Error as err: + sys.exit(f"ERROR: SQLite query failed: {err}") + + if not pg_row: + sys.exit("ERROR: No PostgreSQL version row found.") + + pg_ver = pg_row[0] + spock_ver, spock_comp = (sp_row or (None, None)) + + # Already on Spock 5? No-op. + if spock_ver and ( + _SPOCK5_NAME_RE.match(spock_comp) or _SPOCK5_VER_RE.match(spock_ver) + ): + return 0 + + # Downtime warning + banner = "=" * 80 + + if not spock_ver: + # Case 1: no existing Spock installed, run validation but don't print banner + try: + util.validate_spock_pg_compat('50', pg_ver) + except Exception as exc: + sys.exit(f"ERROR: Compatibility check failed: {exc}") + + elif spock_ver.startswith('4'): + # Case 2: Spock 4.x installed, this is a major upgrade + print(f"\n{banner}") + print("*** WARNING: This operation will cause downtime! ***") + print(f"{banner}\n") + print(f"Detected Spock version {spock_ver} on PostgreSQL {pg_ver}") + try: + util.validate_spock_pg_compat(spock_ver, pg_ver) + except Exception as exc: + sys.exit(f"ERROR: Compatibility check failed: {exc}") + + print("Compatibility check passed.") + return 0 if __name__ == "__main__": fire.Fire( From a32a41bf93d44677912d287df2825065c4894483 Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Fri, 25 Jul 2025 07:19:25 -0500 Subject: [PATCH 32/42] update python deps minor versions (#355) --- requirements.txt | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/requirements.txt b/requirements.txt index 604ab605..8f9a6261 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,41 +1,41 @@ ## Core CLI ######### -requests==2.32.2 -paramiko==3.5.0 -psycopg==3.2.3; platform_system=="Linux" -psycopg-binary==3.2.3; platform_system=="Linux" -psutil==6.1.0 +requests==2.32.4 +paramiko==3.5.1 +psycopg==3.2.9; platform_system=="Linux" +psycopg-binary==3.2.9; platform_system=="Linux" +psutil==6.1.1 pypsutil==0.2.0 semantic_version==2.10.0 -six==1.16.0 +six==1.17.0 termcolor==2.5.0 -typing_extensions==4.12.2 -click==8.1.7 +typing_extensions==4.14.1 +click==8.2.1 tabulate==0.9.0 -python-crontab==3.2.0 -certifi==2024.7.4 -charset_normalizer==3.3.2 +python-crontab==3.3.0 +certifi==2024.12.14 +charset_normalizer==3.4.2 gpustat==1.1.1 py-cpuinfo==9.0.0 -prettytable==3.12.0 -Flask==3.0.3 -minio==7.2.10 -tqdm==4.67.0 +prettytable==3.16.0 +Flask==3.1.1 +minio==7.2.15 +tqdm==4.67.1 ## ACE Advanced ##### ordered-set==4.1.0 mpire[dashboard]==2.10.2 rich==13.9.4 -apscheduler==3.10.4 -pandas==2.2.3 +apscheduler==3.11.0 +pandas==2.3.1 pyopenssl==24.3.0 -packaging==23.1 +packaging==23.2 ## pgEdge-HA ######## cdiff==1.0 -urllib3==2.2.2 -PyYAML==6.0.1 +urllib3==2.5.0 +PyYAML==6.0.2 python-etcd==0.4.5 python-dateutil==2.9.0 -cryptography==44.0.1 -ydiff==1.3 +cryptography==44.0.3 +ydiff==1.4.2 From 96cff32201ce9c1052312f122f6a2f1adb810447 Mon Sep 17 00:00:00 2001 From: hayee-bhatti Date: Wed, 30 Jul 2025 16:29:07 +0500 Subject: [PATCH 33/42] PLAT-184 remove an unused dependency --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8f9a6261..0c521bfc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,6 @@ semantic_version==2.10.0 six==1.17.0 termcolor==2.5.0 typing_extensions==4.14.1 -click==8.2.1 tabulate==0.9.0 python-crontab==3.3.0 certifi==2024.12.14 From 7dc6f85000fcfae2f805c8490ed1dfcc42fa6f8d Mon Sep 17 00:00:00 2001 From: hayee-bhatti Date: Wed, 30 Jul 2025 16:31:57 +0500 Subject: [PATCH 34/42] PLAT-192 Bump ctlibs version to 1.7 --- env.sh | 2 +- src/conf/versions.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/env.sh b/env.sh index 07a0eb0a..c7ee9792 100755 --- a/env.sh +++ b/env.sh @@ -6,7 +6,7 @@ kirkV=$hubV bundle=pgedge api=pgedge -ctlibsV=1.6 +ctlibsV=1.7 spock50V=5.0.0-1 diff --git a/src/conf/versions.sql b/src/conf/versions.sql index f10e9127..6c31e133 100644 --- a/src/conf/versions.sql +++ b/src/conf/versions.sql @@ -405,7 +405,7 @@ INSERT INTO versions VALUES ('hypopg-pg16', '1.4.1-1', 'amd, arm', 1, '20230509 INSERT INTO projects VALUES ('ctlibs', 'pge', 0, 0, '', 3, 'https://github.com/pgedge/cli', 'ctlibs', 0, 'ctlibs.png', 'ctlibs', 'https://github.com/pgedge/cli', ''); INSERT INTO releases VALUES ('ctlibs', 2, 'ctlibs', 'pgEdge Libs', '', 'prod', '', 1, '', '', ''); -INSERT INTO versions VALUES ('ctlibs', '1.6', '', 1, '20240925', '', '', ''); +INSERT INTO versions VALUES ('ctlibs', '1.7', '', 1, '20250729', '', '', ''); -- ## PGCAT ############################# INSERT INTO projects VALUES ('pgcat', 'pge', 11, 5433, '', 3, 'https://github.com/pgedge/pgcat/tags', From 4753c68f84a19dd74048ab7aac1d8a10edadd6f5 Mon Sep 17 00:00:00 2001 From: Tej Kashi Date: Fri, 1 Aug 2025 13:27:11 -0400 Subject: [PATCH 35/42] Merge pull request #360 from pgEdge/ace/datatype-fix * Use conservative datatype handling * Add configurable option for using repeatable read while updating Merkle trees --- cli/scripts/ace-tests/test_data_types.py | 632 +++++++++++++++++++---- cli/scripts/ace.py | 129 +++-- cli/scripts/ace_config.py | 3 + cli/scripts/ace_core.py | 8 +- cli/scripts/ace_mtree.py | 9 +- 5 files changed, 594 insertions(+), 187 deletions(-) diff --git a/cli/scripts/ace-tests/test_data_types.py b/cli/scripts/ace-tests/test_data_types.py index 950bf2dd..ff5d7aec 100644 --- a/cli/scripts/ace-tests/test_data_types.py +++ b/cli/scripts/ace-tests/test_data_types.py @@ -1,3 +1,6 @@ +from datetime import datetime, timedelta, date, time +from decimal import Decimal +from ipaddress import IPv4Address import pytest import psycopg import json @@ -31,50 +34,25 @@ def setup_datatypes(self, nodes): bytea_col BYTEA, point_col POINT, text_col TEXT, - text_array_col TEXT[] + text_array_col TEXT[], + bool_col BOOLEAN, + bigint_col BIGINT, + smallint_col SMALLINT, + numeric_col NUMERIC(10, 4), + real_col REAL, + time_col TIME, + date_col DATE, + timestamp_col TIMESTAMP, + interval_col INTERVAL, + inet_col INET, + macaddr_col MACADDR, + money_col MONEY ) """ ) # Insert sample data - cur.execute( - """ - INSERT INTO datatypes_test VALUES - ( - 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', - 42, - 3.14159, - ARRAY[1, 2, 3, 4, 5], - '{"key": "value", "nested": {"foo": "bar"}}', - decode('DEADBEEF', 'hex'), - point(1.5, 2.5), - 'sample text', - ARRAY['apple', 'banana', 'cherry'] - ), - ( - 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12', - 100, - 2.71828, - ARRAY[10, 20, 30], - '{"numbers": [1, 2, 3], "active": true}', - decode('BADDCAFE', 'hex'), - point(3.7, 4.2), - 'another sample', - ARRAY['dog', 'cat', 'bird'] - ), - ( - 'c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a13', - -17, - 0.577216, - ARRAY[]::INTEGER[], - '{"empty": true}', - NULL, - point(0, 0), - 'third sample', - ARRAY[]::TEXT[] - ) - """ - ) + self._insert_initial_data(cur) repset_add_datatypes_sql = """ SELECT spock.repset_add_table('test_repset', 'datatypes_test') @@ -106,6 +84,145 @@ def setup_datatypes(self, nodes): except Exception as e: pytest.fail(f"Failed to setup/cleanup datatypes test: {str(e)}") + def _insert_initial_data(self, cur): + """Helper method to insert the initial dataset.""" + cur.execute(self._get_initial_data_sql()) + + def _get_initial_data_sql(self): + """Returns the SQL for inserting the initial dataset.""" + return """ + INSERT INTO datatypes_test VALUES + ( + 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', + 42, + 3.14159, + ARRAY[1, 2, 3, 4, 5], + '{"key": "value", "nested": {"foo": "bar"}}', + decode('DEADBEEF', 'hex'), + point(1.5, 2.5), + 'sample text', + ARRAY['apple', 'banana', 'cherry'], + true, + 9223372036854775807, + 32767, + 12345.6789, + 123.456, + '12:34:56', + '2024-01-01', + '2024-01-01 12:34:56', + '30 days', + '192.168.1.1', + '08:00:2b:01:02:03', + 12345.67 + ), + ( + 'b0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12', + 100, + 2.71828, + ARRAY[10, 20, 30], + '{"numbers": [1, 2, 3], "active": true}', + decode('BADDCAFE', 'hex'), + point(3.7, 4.2), + 'another sample', + ARRAY['dog', 'cat', 'bird'], + false, + -9223372036854775808, + -32768, + -12345.6789, + -123.456, + '23:59:59', + '2023-12-31', + '2023-12-31 23:59:59', + '-5 days', + '10.0.0.1', + '00:1A:2B:3C:4D:5E', + -12345.67 + ), + ( + 'c0eebc99-9c0b-4ef8-bb6d-6bb9bd380a13', + -17, + 0.577216, + ARRAY[]::INTEGER[], + '{"empty": true}', + NULL, + point(0, 0), + 'third sample', + ARRAY[]::TEXT[], + true, + 0, + 0, + 0, + 0, + '00:00:00', + '1970-01-01', + '1970-01-01 00:00:00', + '0 seconds', + '0.0.0.0', + '00:00:00:00:00:00', + 0 + ), + ( + 'd0eebc99-9c0b-4ef8-bb6d-6bb9bd380a14', + NULL, + NULL, + ARRAY[NULL, 1, NULL], + NULL, + NULL, + NULL, + 'null', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ), + ( + 'e0eebc99-9c0b-4ef8-bb6d-6bb9bd380a15', + 1, + 'NaN', + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL, + NULL + ) + """ + + def reset_data(self, nodes): + """Reset data in the test table before each test function.""" + try: + for node in nodes: + conn = psycopg.connect(host=node, dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("TRUNCATE TABLE datatypes_test") + self._insert_initial_data(cur) + conn.commit() + cur.close() + conn.close() + except Exception as e: + pytest.fail(f"Failed to reset data between tests: {str(e)}") + # Override the table_name parameter for all parameterized tests @pytest.mark.parametrize("table_name", ["public.datatypes_test"]) def test_simple_table_diff(self, cli, capsys, table_name): @@ -113,16 +230,28 @@ def test_simple_table_diff(self, cli, capsys, table_name): @pytest.mark.parametrize("table_name", ["public.datatypes_test"]) @pytest.mark.parametrize( - "column_name,test_value", + "column_name,test_value,expected_diffs", [ - ("int_col", "9999"), - ("float_col", "123.456"), - ("array_col", "ARRAY[99, 98, 97]"), - ("json_col", '\'{"test": "modified"}\''), - ("bytea_col", "decode('FEEDFACE', 'hex')"), - ("point_col", "point(99.9, 99.9)"), - ("text_col", "'modified-text'"), - ("text_array_col", "ARRAY['modified', 'text', 'array']"), + ("int_col", "9999", 5), + ("float_col", "123.456", 5), + ("array_col", "ARRAY[99, 98, 97]", 5), + ("json_col", '\'{"test": "modified"}\'', 5), + ("bytea_col", "decode('FEEDFACE', 'hex')", 5), + ("point_col", "point(99.9, 99.9)", 5), + ("text_col", "'modified-text'", 5), + ("text_array_col", "ARRAY['modified', 'text', 'array']", 5), + ("bool_col", "false", 5), + ("bigint_col", "1234567890123456789", 5), + ("smallint_col", "-32768", 5), + ("numeric_col", "98765.4321", 5), + ("real_col", "987.654", 5), + ("time_col", "'11:22:33'", 5), + ("date_col", "'2025-05-25'", 5), + ("timestamp_col", "'2025-05-25 11:22:33'", 5), + ("interval_col", "'90 days'", 5), + ("inet_col", "'192.168.100.200'", 5), + ("macaddr_col", "'01:23:45:67:89:ab'", 5), + ("money_col", "9876.54", 5), ], ) @pytest.mark.parametrize("key_column", ["id"]) @@ -134,6 +263,7 @@ def test_table_diff_with_differences( table_name, column_name, test_value, + expected_diffs, key_column, diff_file_path, ): @@ -154,11 +284,6 @@ def test_table_diff_with_differences( """ ) - modified_rows = cur.fetchall() - modified_indices = { - str(row[0]) for row in modified_rows - } # Convert UUID to string - conn.commit() cur.close() conn.close() @@ -185,55 +310,79 @@ def test_table_diff_with_differences( # Verify number of differences assert ( - len(diff_data["diffs"]["n1/n2"]["n2"]) == 3 - ), "Expected 3 differences," + len(diff_data["diffs"]["n1/n2"]["n2"]) == expected_diffs + ), f"Expected {expected_diffs} differences," f" found {len(diff_data['diffs']['n1/n2']['n2'])}" - # Verify modified rows are in diff - diff_indices = { - str(diff["id"]) for diff in diff_data["diffs"]["n1/n2"]["n2"] - } - assert ( - modified_indices == diff_indices - ), "Modified rows don't match diff file records" - # Verify the differences are correctly reported for diff in diff_data["diffs"]["n1/n2"]["n2"]: + diff_val = diff[column_name] + expected_val_str = test_value.strip("'") + if column_name == "json_col": assert ( - diff[column_name].get("test") == "modified" + diff_val.get("test") == "modified" ), f"Modified row {diff['id']} doesn't have expected JSON value" elif column_name == "array_col": - assert diff[column_name] == [ + assert diff_val == [ 99, 98, 97, ], f"Modified row {diff['id']} doesn't have expected array value" elif column_name == "text_array_col": - assert diff[column_name] == [ + assert diff_val == [ "modified", "text", "array", ], ( - f"Modified row {diff['id']} doesn't have expected " - "text array value" + f"Modified row {diff['id']} doesn't have expected text" + " array value" ) elif column_name == "point_col": assert ( - diff[column_name] == "(99.9,99.9)" + diff_val == "(99.9,99.9)" ), f"Modified row {diff['id']} doesn't have expected point value" elif column_name == "bytea_col": - print("bytea col: ", diff[column_name]) assert ( - diff[column_name] == "feedface" + diff_val == "feedface" ), f"Modified row {diff['id']} doesn't have expected bytea value" + elif column_name == "macaddr_col": + assert ( + diff_val == "01:23:45:67:89:ab" + ), f"Modified row {diff['id']} doesn't have expected macaddr value" + elif column_name == "money_col": + cleaned_diff = diff_val.replace("$", "").replace(",", "") + assert float(cleaned_diff) == float( + expected_val_str + ), f"Modified row {diff['id']} doesn't have expected money value" + elif column_name == "bool_col": + assert str(diff_val).lower() == expected_val_str.lower(), ( + f"Modified row {diff['id']} " + "doesn't have expected boolean value" + ) + elif column_name == "interval_col": + # Interval representation can vary, so we check equality + # directly in Postgres + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + "SELECT %s::interval = %s::interval", + (str(diff_val), expected_val_str), + ) + is_equal = cur.fetchone()[0] + cur.close() + conn.close() + assert is_equal, ( + f"Modified row {diff['id']} " + f"doesn't have expected interval value. " + f"Got {diff_val}, expected equivalence to {expected_val_str}" + ) else: - assert str(diff[column_name]) in ( - test_value.strip("'"), - "9999", - "123.456", - "modified-text", - ), f"Modified row {diff['id']} doesn't have expected value" + assert str(diff_val) == expected_val_str, ( + f"Modified row {diff['id']} " + f"doesn't have expected value, got {diff_val} " + f"expected {expected_val_str}" + ) except Exception as e: pytest.fail(f"Failed to test differences for {column_name}: {str(e)}") @@ -245,16 +394,28 @@ def test_simple_table_repair(self, cli, capsys, table_name, diff_file_path): @pytest.mark.parametrize("table_name", ["public.datatypes_test"]) @pytest.mark.parametrize("key_column", ["id"]) @pytest.mark.parametrize( - "column_name,test_value", + "column_name,test_value,expected_rerun_diffs", [ - ("int_col", "1234"), - ("float_col", "98.765"), - ("array_col", "ARRAY[11, 22, 33]"), - ("json_col", '\'{"rerun": "modified"}\''), - ("bytea_col", "decode('ABCDEF12', 'hex')"), - ("point_col", "point(88.8, 88.8)"), - ("text_col", "'rerun-modified'"), - ("text_array_col", "ARRAY['rerun', 'modified', 'array']"), + ("int_col", "1234", 5), + ("float_col", "98.765", 5), + ("array_col", "ARRAY[11, 22, 33]", 5), + ("json_col", '\'{"rerun": "modified"}\'', 5), + ("bytea_col", "decode('ABCDEF12', 'hex')", 5), + ("point_col", "point(88.8, 88.8)", 5), + ("text_col", "'rerun-modified'", 5), + ("text_array_col", "ARRAY['rerun', 'modified', 'array']", 5), + ("bool_col", "true", 5), + ("bigint_col", "-1234567890123456789", 5), + ("smallint_col", "32767", 5), + ("numeric_col", "-98765.4321", 5), + ("real_col", "-987.654", 5), + ("time_col", "'01:02:03'", 5), + ("date_col", "'2022-02-02'", 5), + ("timestamp_col", "'2022-02-02 01:02:03'", 5), + ("interval_col", "'60 days'", 5), + ("inet_col", "'127.0.0.1'", 5), + ("macaddr_col", "'fe:dc:ba:98:76:54'", 5), + ("money_col", "-9876.54", 5), ], ) def test_table_rerun_temptable( @@ -266,6 +427,7 @@ def test_table_rerun_temptable( key_column, column_name, test_value, + expected_rerun_diffs, diff_file_path, ): """Test table rerun temptable with various data types""" @@ -323,42 +485,115 @@ def test_table_rerun_temptable( diff_data = json.load(f) assert ( - len(diff_data["diffs"]["n1/n2"]["n2"]) == 3 - ), f"Expected 3 differences, found {len(diff_data['diffs']['n1/n2']['n2'])}" + len(diff_data["diffs"]["n1/n2"]["n2"]) == expected_rerun_diffs + ), f"Expected {expected_rerun_diffs} differences, " + f"found {len(diff_data['diffs']['n1/n2']['n2'])}" # Verify the differences are correctly reported for diff in diff_data["diffs"]["n1/n2"]["n2"]: + diff_val = diff[column_name] + expected_val_str = test_value.strip("'") + if column_name == "json_col": assert ( - diff[column_name].get("rerun") == "modified" + diff_val.get("rerun") == "modified" ), f"Modified row {diff['id']} doesn't have expected JSON value" elif column_name == "array_col": - assert diff[column_name] == [ + assert diff_val == [ 11, 22, 33, ], f"Modified row {diff['id']} doesn't have expected array value" elif column_name == "text_array_col": - assert diff[column_name] == [ + assert diff_val == [ "rerun", "modified", "array", ], f"Modified row {diff['id']} doesn't have expected text array value" elif column_name == "point_col": assert ( - diff[column_name] == "(88.8,88.8)" + diff_val == "(88.8,88.8)" ), f"Modified row {diff['id']} doesn't have expected point value" elif column_name == "bytea_col": assert ( - diff[column_name] == "abcdef12" + diff_val == "abcdef12" ), f"Modified row {diff['id']} doesn't have expected bytea value" + elif column_name == "macaddr_col": + assert ( + diff_val == "fe:dc:ba:98:76:54" + ), f"Modified row {diff['id']} doesn't have expected macaddr value" + elif column_name == "money_col": + cleaned_diff = diff_val.replace("$", "").replace(",", "") + assert float(cleaned_diff) == float( + expected_val_str + ), f"Modified row {diff['id']} doesn't have expected money value" + elif column_name == "bool_col": + assert ( + str(diff_val).lower() == expected_val_str.lower() + ), f"Modified row {diff['id']} doesn't have expected boolean value" + elif column_name == "interval_col": + # Interval representation can vary, so we check equality in the DB + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + "SELECT %s::interval = %s::interval", + (str(diff_val), expected_val_str), + ) + is_equal = cur.fetchone()[0] + cur.close() + conn.close() + assert is_equal, ( + f"Modified row {diff['id']} " + f"doesn't have expected interval value. " + f"Got {diff_val}, expected equivalence to {expected_val_str}" + ) else: - assert str(diff[column_name]) in ( - test_value.strip("'"), - "1234", - "98.765", - "rerun-modified", - ), f"Modified row {diff['id']} doesn't have expected value" + assert str(diff_val) == expected_val_str, ( + f"Modified row {diff['id']} " + f"doesn't have expected value, got {diff_val} " + f"expected {expected_val_str}" + ) + + def _verify_repaired_value(self, column_name, repaired_value, expected_value): + """Helper function to verify repaired values based on data type""" + if column_name == "bytea_col": + assert ( + repaired_value == expected_value + ), "Repaired bytea value doesn't match expected value" + elif column_name == "point_col": + if isinstance(repaired_value, str): + repaired_tuple = tuple( + map(float, repaired_value.strip("()").split(",")) + ) + else: + repaired_tuple = repaired_value + + expected_tuple = tuple(map(float, expected_value.strip("()").split(","))) + assert ( + repaired_tuple == expected_tuple + ), "Repaired point value doesn't match expected value" + elif column_name == "numeric_col": + assert repaired_value == Decimal( + expected_value + ), f"Repaired value for {column_name} doesn't match" + elif column_name == "inet_col": + assert repaired_value == IPv4Address( + expected_value + ), f"Repaired value for {column_name} doesn't match" + elif column_name in ["time_col", "date_col", "timestamp_col", "interval_col"]: + assert ( + repaired_value == expected_value + ), f"Repaired value for {column_name} doesn't match" + elif column_name == "money_col": + cleaned_repaired = str(repaired_value).replace("$", "").replace(",", "") + cleaned_expected = str(expected_value).replace("$", "").replace(",", "") + assert float(cleaned_repaired) == float( + cleaned_expected + ), f"Repaired money value doesn't match for {column_name}" + else: + assert ( + repaired_value == expected_value + ), f"Repaired value doesn't match expected value for {column_name}" @pytest.mark.parametrize("table_name", ["public.datatypes_test"]) @pytest.mark.parametrize( @@ -376,6 +611,22 @@ def test_table_rerun_temptable( "ARRAY['modified', 'text', 'array']", ["modified", "text", "array"], ), + ("bool_col", "false", False), + ("bigint_col", "1234567890123456789", 1234567890123456789), + ("smallint_col", "-32768", -32768), + ("numeric_col", "98765.4321", Decimal("98765.4321")), + ("real_col", "987.654", 987.654), + ("time_col", "'11:22:33'", time(11, 22, 33)), + ("date_col", "'2025-05-25'", date(2025, 5, 25)), + ( + "timestamp_col", + "'2025-05-25 11:22:33'", + datetime(2025, 5, 25, 11, 22, 33), + ), + ("interval_col", "'90 days'", timedelta(days=90)), + ("inet_col", "'192.168.100.200'", "192.168.100.200"), + ("macaddr_col", "'01:23:45:67:89:ab'", "01:23:45:67:89:ab"), + ("money_col", "9876.54", "$9,876.54"), ], ) def test_table_repair_datatypes( @@ -441,18 +692,177 @@ def test_table_repair_datatypes( conn.close() # Compare with expected value - if column_name == "bytea_col": - assert ( - repaired_value == expected_value - ), "Repaired bytea value doesn't match expected value" - elif column_name == "point_col": - assert ( - str(repaired_value) == expected_value - ), "Repaired point value doesn't match expected value" - else: - assert ( - repaired_value == expected_value - ), f"Repaired value doesn't match expected value for {column_name}" + self._verify_repaired_value(column_name, repaired_value, expected_value) except Exception as e: pytest.fail(f"Test failed: {str(e)}") + + @pytest.mark.parametrize("id_to_update", ["d0eebc99-9c0b-4ef8-bb6d-6bb9bd380a14"]) + def test_null_and_string_literal_handling( + self, cli, capsys, diff_file_path, id_to_update + ): + """ + Verify that NULL values and string literals like 'null' are + handled correctly. + """ + try: + # Our prior repair unfortunately reset a lot of fields, so we reset + # the data first here + self.reset_data(nodes=["n1", "n2"]) + + # On n2, update text_col from 'null' to 'not null' and + # int_col from NULL to a number + conn = psycopg.connect(host="n2", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + # This specific id has "null" as a literal in the text_col + cur.execute( + """ + UPDATE datatypes_test + SET text_col = 'not null anymore', int_col = 123 + WHERE id = %s + """, + (id_to_update,), + ) + conn.commit() + cur.close() + conn.close() + + # Run table-diff + cli.table_diff(cluster_name="eqn-t9da", table_name="public.datatypes_test") + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file_path.path = match.group(1) + + # Verify diff file + with open(diff_file_path.path, "r") as f: + diff_data = json.load(f) + + diffs_n1 = diff_data["diffs"]["n1/n2"]["n1"] + diffs_n2 = diff_data["diffs"]["n1/n2"]["n2"] + + assert len(diffs_n1) == 1, "Expected 1 difference on n1" + assert len(diffs_n2) == 1, "Expected 1 difference on n2" + + # Check n1 (original values) + assert diffs_n1[0]["id"] == id_to_update + assert diffs_n1[0]["text_col"] == "null" + assert diffs_n1[0]["int_col"] is None + + # Check n2 (modified values) + assert diffs_n2[0]["id"] == id_to_update + assert diffs_n2[0]["text_col"] == "not null anymore" + assert diffs_n2[0]["int_col"] == 123 + + # Run table-repair + cli.table_repair( + cluster_name="eqn-t9da", + table_name="public.datatypes_test", + diff_file=diff_file_path.path, + source_of_truth="n2", + ) + + # Verify repair on n1 + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + """ + SELECT text_col, int_col FROM datatypes_test + WHERE id = %s + """, + (id_to_update,), + ) + repaired_text, repaired_int = cur.fetchone() + cur.close() + conn.close() + + assert repaired_text == "not null anymore" + assert repaired_int == 123 + + except Exception as e: + pytest.fail(f"Test for null handling failed: {str(e)}") + + @pytest.mark.parametrize("id_to_update", ["e0eebc99-9c0b-4ef8-bb6d-6bb9bd380a15"]) + def test_ast_literal_eval_fallback(self, cli, capsys, diff_file_path, id_to_update): + """ + Verify that the fallback to string representation works when + ast.literal_eval fails. + """ + try: + # Resetting again here + self.reset_data(nodes=["n1", "n2"]) + + # On n2, update float_col from NaN to a valid number + conn = psycopg.connect(host="n2", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute("SELECT spock.repair_mode(true)") + cur.execute( + """ + UPDATE datatypes_test + SET float_col = 1.23 + WHERE id = %s + """, + (id_to_update,), + ) + conn.commit() + cur.close() + conn.close() + + # Run table-diff + cli.table_diff(cluster_name="eqn-t9da", table_name="public.datatypes_test") + captured = capsys.readouterr() + clean_output = re.sub( + r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])", "", captured.out + ) + match = re.search(r"diffs written out to (.+\.json)", clean_output.lower()) + assert match, "Diff file path not found in output" + diff_file_path.path = match.group(1) + + # Verify diff file + with open(diff_file_path.path, "r") as f: + diff_data = json.load(f) + + diffs_n1 = diff_data["diffs"]["n1/n2"]["n1"] + diffs_n2 = diff_data["diffs"]["n1/n2"]["n2"] + + assert len(diffs_n1) == 1, "Expected 1 difference on n1" + assert len(diffs_n2) == 1, "Expected 1 difference on n2" + + # Check n1 (original 'NaN' value) + assert diffs_n1[0]["id"] == id_to_update + assert diffs_n1[0]["float_col"] == "nan" + + # Check n2 (modified value) + assert diffs_n2[0]["id"] == id_to_update + assert diffs_n2[0]["float_col"] == 1.23 + + # Run table-repair + cli.table_repair( + cluster_name="eqn-t9da", + table_name="public.datatypes_test", + diff_file=diff_file_path.path, + source_of_truth="n2", + ) + + # Verify repair on n1 + conn = psycopg.connect(host="n1", dbname="demo", user="admin") + cur = conn.cursor() + cur.execute( + """ + SELECT float_col FROM datatypes_test + WHERE id = %s + """, + (id_to_update,), + ) + repaired_float = cur.fetchone()[0] + cur.close() + conn.close() + + assert repaired_float == 1.23 + + except Exception as e: + pytest.fail(f"Test for ast.literal_eval fallback failed: {str(e)}") diff --git a/cli/scripts/ace.py b/cli/scripts/ace.py index ecf50d43..0fed3a05 100755 --- a/cli/scripts/ace.py +++ b/cli/scripts/ace.py @@ -727,63 +727,47 @@ def check_diff_file_format(diff_file_path: str, task) -> dict: return diff_json -def convert_pg_type_to_json(item: str, type: str): +def convert_pg_type_to_json(item, type): """ Converts a value from a postgres column to a json-compatible type. """ # TODO: Need to revisit this. - try: - # List of types that should be treated as strings - string_types = [ - "char", - "text", - "time", - "bytea", - "uuid", - "date", - "timestamp", - "interval", - "inet", - "macaddr", - "xml", - "money", - "point", - "line", - "polygon", - ] - - # Types that can be directly represented in JSON - json_compatible_types = [ - "json", - "jsonb", - "boolean", - "integer", - "bigint", - "smallint", - "numeric", - "real", - "double precision", - ] - - type_lower = type.lower() - - if not item or item == "" or item.lower() == "null" or item.lower() == "none": - return None - elif "[]" in type_lower: - return ast.literal_eval(item) - elif any(s in type_lower for s in json_compatible_types): - # For JSON-compatible types, parse them using AST - return ast.literal_eval(item) - elif any(s in type_lower for s in string_types): - return item - else: - # Default to treating as string if type is unknown - return item + type_lower = type.lower() - except Exception as e: - raise AceException( - f"Could not convert value {item} to {type} while writing to json: {e}" - ) + # Types that should be parsed into native JSON types (not strings) + json_compatible_types = [ + "json", + "jsonb", + "boolean", + "integer", + "bigint", + "smallint", + "numeric", + "real", + "double precision", + ] + + is_parsable = ( + any(s in type_lower for s in json_compatible_types) or "[]" in type_lower + ) + + if not is_parsable: + # For string-like types (text, varchar, etc.), we return the value + # directly. This correctly preserves string literals like 'None' or + # 'null'. A database NULL would arrive here as item=None from the driver. + return item + + # For parsable types (numeric, boolean, json, array), we can interpret + # 'null' and 'none'. + if item is None or str(item).lower() in ("", "null", "none"): + return None + + try: + # For JSON-compatible types, parse them using AST + return ast.literal_eval(str(item)) + except (ValueError, SyntaxError): + # If conversion fails, treat as a string + return str(item) def convert_json_to_pg_type(rows, cols_list, col_types) -> list[tuple]: @@ -841,31 +825,40 @@ def convert_json_to_pg_type(rows, cols_list, col_types) -> list[tuple]: modified_row = tuple() for col_name in cols_list: col_type = col_types[col_name] - elem = str(row[col_name]) + elem = row[col_name] + type_lower = col_type.lower() try: - type_lower = col_type.lower() - - if ( - not elem - or elem == "" - or elem.lower() == "null" - or elem.lower() == "none" - ): - modified_row += (None,) - elif "[]" in type_lower: - modified_row += (ast.literal_eval(elem),) - elif any(s in type_lower for s in string_types): + # If the column type is a string type, we don't need to do anything + # special. A value of None will be converted to NULL by psycopg. + if any(s in type_lower for s in string_types): if type_lower == "bytea": - modified_row += (bytes.fromhex(elem),) + # We stored bytea as hex, so we need to convert it back + if elem is not None: + modified_row += (bytes.fromhex(elem),) + else: + modified_row += (None,) else: modified_row += (elem,) + continue + + # For non-string types, if the value is None, or looks like null, + # it should be treated as such. + if elem is None or str(elem).lower() in ("null", "none", ""): + modified_row += (None,) + continue + + elem_str = str(elem) + + if "[]" in type_lower: + modified_row += (ast.literal_eval(elem_str),) elif any(s in type_lower for s in json_compatible_types): - item = ast.literal_eval(elem) - if type_lower == "jsonb" or type_lower == "json": + item = ast.literal_eval(elem_str) + if type_lower in ("jsonb", "json"): item = json.dumps(item) modified_row += (item,) else: + # Fallback for any other types modified_row += (elem,) except (ValueError, SyntaxError): diff --git a/cli/scripts/ace_config.py b/cli/scripts/ace_config.py index 1580b1c4..39379e0e 100644 --- a/cli/scripts/ace_config.py +++ b/cli/scripts/ace_config.py @@ -12,6 +12,9 @@ STATEMENT_TIMEOUT = 0 # in milliseconds CONNECTION_TIMEOUT = 10 # in seconds +# Whether to use repeatable read isolation for Merkle tree updates +USE_REPEATABLE_READ = False + # Default values for ACE table-diff MAX_DIFF_ROWS = 1_000_000 MIN_DIFF_BLOCK_SIZE = 1000 diff --git a/cli/scripts/ace_core.py b/cli/scripts/ace_core.py index 91567668..b0c35b45 100644 --- a/cli/scripts/ace_core.py +++ b/cli/scripts/ace_core.py @@ -574,12 +574,16 @@ def compare_checksums(worker_id, shared_objects, worker_state, pkey1, pkey2): for row_key in t1_only: worker_diffs[node_pair_key][host1].append( - dict(zip(cols, (str(x) for x in row_key))) + dict( + zip(cols, (str(x) if x is not None else None for x in row_key)) + ) ) for row_key in t2_only: worker_diffs[node_pair_key][host2].append( - dict(zip(cols, (str(x) for x in row_key))) + dict( + zip(cols, (str(x) if x is not None else None for x in row_key)) + ) ) total_diffs += max(len(t1_only), len(t2_only)) diff --git a/cli/scripts/ace_mtree.py b/cli/scripts/ace_mtree.py index a621b66b..32eb7980 100644 --- a/cli/scripts/ace_mtree.py +++ b/cli/scripts/ace_mtree.py @@ -1025,7 +1025,6 @@ def split_blocks(conn, schema, table, key, blocks, block_size): i += 1 - conn.commit() pbar.close() return list(modified_positions) @@ -1399,7 +1398,6 @@ def merge_blocks(conn, schema, table, key, blocks, block_size): if i >= len(blocks): break - conn.commit() pbar.close() return list(modified_positions) @@ -1408,7 +1406,7 @@ def update_mtree(mtree_task: MerkleTreeTask, skip_all_checks=False) -> None: """ Update a Merkle tree by recomputing hashes for dirty leaf nodes and new blocks. Also processes any pending block rebalancing operations. - Uses repeatable read isolation to ensure consistency during the update. + Uses repeatable read isolation if config.USE_REPEATABLE_READ is True. Args: cluster_name (str): Name of the cluster @@ -1455,7 +1453,8 @@ def update_mtree(mtree_task: MerkleTreeTask, skip_all_checks=False) -> None: for node in mtree_task.fields.cluster_nodes: _, conn = mtree_task.connection_pool.connect(node) - conn.set_isolation_level(IsolationLevel.REPEATABLE_READ) + if config.USE_REPEATABLE_READ: + conn.set_isolation_level(IsolationLevel.REPEATABLE_READ) print(f"\nUpdating Merkle tree on node: {node['name']}") @@ -1476,7 +1475,6 @@ def update_mtree(mtree_task: MerkleTreeTask, skip_all_checks=False) -> None: if not blocks_to_update: print(f"No updates needed for {node['name']}") - conn.commit() continue # First identify blocks that might need splitting based on insert count @@ -1557,7 +1555,6 @@ def update_mtree(mtree_task: MerkleTreeTask, skip_all_checks=False) -> None: if not blocks_to_update: print(f"No updates needed for {node['name']}") - conn.commit() continue print(f"Found {len(blocks_to_update)} blocks to update") From 5ba9c8fc04f13d3abc3616288ac5433675ab9302 Mon Sep 17 00:00:00 2001 From: Hayee Bhatti <152845623+hayee-bhatti@users.noreply.github.com> Date: Fri, 1 Aug 2025 23:20:10 +0500 Subject: [PATCH 36/42] [BR-158]: Introduce spock60 builds in CLI (#362) Build script changes to produce spock60 builds from the main branch. Updated versioning (6.0.0-devel) and metadata. Updated 'current' builds workflow to now produce spock60 from main. Although, spock60 is hidden in the um list output, it can be installed in the current builds (only) by passing --spock_ver=6.0.0 in the pgedge setup command. --- .../workflows/current-amd8-daily-build-devel.yml | 8 ++++---- .../workflows/current-arm9-daily-build-devel.yml | 8 ++++---- build.sh | 3 +++ env.sh | 2 ++ src/conf/versions.sql | 14 ++++++++++++++ 5 files changed, 27 insertions(+), 8 deletions(-) diff --git a/.github/workflows/current-amd8-daily-build-devel.yml b/.github/workflows/current-amd8-daily-build-devel.yml index 76a334ee..08b772be 100644 --- a/.github/workflows/current-amd8-daily-build-devel.yml +++ b/.github/workflows/current-amd8-daily-build-devel.yml @@ -4,7 +4,7 @@ name: Current Daily Build Devel - amd8 env: DEFAULT_CLI_BRANCH: "main" # Default CLI branch for scheduled runs DEFAULT_MODE: "current" # Always "current" for this workflow - DEFAULT_COMPONENT: "spock50" # Default spock component name + DEFAULT_COMPONENT: "spock60" # Default spock component name DEFAULT_BRANCH: "main" # Default branch for the spock component DEFAULT_CLEAN_FLAG: "false" # Default clean flag for scheduled runs @@ -13,7 +13,7 @@ on: workflow_dispatch: inputs: cli_branch: - description: "Select the CLI branch to build from (e.g. v25_STABLE)" + description: "Select the CLI branch to build from (e.g. main)" required: true default: "main" type: choice @@ -23,9 +23,9 @@ on: - REL24_10 component: - description: "Spock in-dev component to additionally build (e.g. spock50)" + description: "Spock in-dev component to additionally build (e.g. spock60)" required: true - default: "spock50" + default: "spock60" type: string branch: diff --git a/.github/workflows/current-arm9-daily-build-devel.yml b/.github/workflows/current-arm9-daily-build-devel.yml index f77a092d..b8c4d645 100644 --- a/.github/workflows/current-arm9-daily-build-devel.yml +++ b/.github/workflows/current-arm9-daily-build-devel.yml @@ -4,7 +4,7 @@ name: Current Daily Build Devel - arm9 env: DEFAULT_CLI_BRANCH: "main" # Default CLI branch for scheduled runs DEFAULT_MODE: "current" # Always "current" for this workflow - DEFAULT_COMPONENT: "spock50" # Default spock component name + DEFAULT_COMPONENT: "spock60" # Default spock component name DEFAULT_BRANCH: "main" # Default branch for the spock component DEFAULT_CLEAN_FLAG: "false" # Default clean flag for scheduled runs @@ -13,7 +13,7 @@ on: workflow_dispatch: inputs: cli_branch: - description: "Select the CLI branch to build from (e.g. v25_STABLE)" + description: "Select the CLI branch to build from (e.g. main)" required: true default: "main" type: choice @@ -23,9 +23,9 @@ on: - REL24_10 component: - description: "Spock in-dev component to additionally build (e.g. spock50)" + description: "Spock in-dev component to additionally build (e.g. spock60)" required: true - default: "spock50" + default: "spock60" type: string branch: diff --git a/build.sh b/build.sh index e0907f6a..f9a9e022 100755 --- a/build.sh +++ b/build.sh @@ -358,18 +358,21 @@ initPG () { initC "audit-pg$pgM" "audit" "$audit17V" "$outPlat" "postgres/audit" "" "" "nil" initC "hintplan-pg$pgM" "hintplan" "$hint17V" "$outPlat" "postgres/hintplan" "" "" "nil" initC "spock50-pg$pgM" "spock50" "$spock50V" "$outPlat" "postgres/spock50" "" "" "nil" + initC "spock60-pg$pgM" "spock60" "$spock60V" "$outPlat" "postgres/spock60" "" "" "nil" fi if [ "$pgM" == "16" ]; then initC "audit-pg$pgM" "audit" "$audit16V" "$outPlat" "postgres/audit" "" "" "nil" initC "hintplan-pg$pgM" "hintplan" "$hint16V" "$outPlat" "postgres/hintplan" "" "" "nil" initC "spock50-pg$pgM" "spock50" "$spock50V" "$outPlat" "postgres/spock50" "" "" "nil" + initC "spock60-pg$pgM" "spock60" "$spock60V" "$outPlat" "postgres/spock60" "" "" "nil" fi if [ "$pgM" == "15" ]; then initC "audit-pg$pgM" "audit" "$audit15V" "$outPlat" "postgres/audit" "" "" "nil" initC "hintplan-pg$pgM" "hintplan" "$hint15V" "$outPlat" "postgres/hintplan" "" "" "nil" initC "spock50-pg$pgM" "spock50" "$spock50V" "$outPlat" "postgres/spock50" "" "" "nil" + initC "spock60-pg$pgM" "spock60" "$spock60V" "$outPlat" "postgres/spock60" "" "" "nil" fi if [ "$pgM" == "15" ] || [ "$pgM" == "16" ] || [ "$pgM" == "17" ]; then diff --git a/env.sh b/env.sh index c7ee9792..76c3656f 100755 --- a/env.sh +++ b/env.sh @@ -8,6 +8,8 @@ bundle=pgedge api=pgedge ctlibsV=1.7 +spock60V=6.0.0-devel-1 + spock50V=5.0.0-1 spock40V=4.0.10-1 diff --git a/src/conf/versions.sql b/src/conf/versions.sql index 6c31e133..31f04ab5 100644 --- a/src/conf/versions.sql +++ b/src/conf/versions.sql @@ -84,6 +84,11 @@ INSERT INTO extensions VALUES ('spock50', 'spock', 1, 'spock', max_wal_senders=16 | hot_standby_feedback=on | wal_sender_timeout=5s | track_commit_timestamp=on | spock.conflict_resolution=last_update_wins | spock.save_resolutions=on | spock.conflict_log_level=DEBUG'); +INSERT INTO extensions VALUES ('spock60', 'spock', 1, 'spock', + 'wal_level=logical | max_worker_processes=12 | max_replication_slots=16 | + max_wal_senders=16 | hot_standby_feedback=on | wal_sender_timeout=5s | + track_commit_timestamp=on | spock.conflict_resolution=last_update_wins | + spock.save_resolutions=on | spock.conflict_log_level=DEBUG'); INSERT INTO extensions VALUES ('lolor', 'lolor', 0, '', ''); INSERT INTO extensions VALUES ('postgis', 'postgis', 1, 'postgis-3', ''); INSERT INTO extensions VALUES ('setuser', 'set_user', 1, 'set_user', ''); @@ -342,6 +347,15 @@ INSERT INTO versions VALUES ('spock50-pg15', '5.0.0-1', 'amd, arm', 1, '202507 INSERT INTO versions VALUES ('spock50-pg16', '5.0.0-1', 'amd, arm', 1, '20250715', 'pg16', '', ''); INSERT INTO versions VALUES ('spock50-pg17', '5.0.0-1', 'amd, arm', 1, '20250715', 'pg17', '', ''); +-- ## spock60 ########################### +INSERT INTO releases VALUES ('spock60-pg15', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); +INSERT INTO releases VALUES ('spock60-pg16', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); +INSERT INTO releases VALUES ('spock60-pg17', 4, 'spock', 'Spock', '', 'test', '', 1, 'pgEdge Community', '', ''); + +INSERT INTO versions VALUES ('spock60-pg15', '6.0.0-devel-1', 'amd, arm', 1, '20250801', 'pg15', '', ''); +INSERT INTO versions VALUES ('spock60-pg16', '6.0.0-devel-1', 'amd, arm', 1, '20250801', 'pg16', '', ''); +INSERT INTO versions VALUES ('spock60-pg17', '6.0.0-devel-1', 'amd, arm', 1, '20250801', 'pg17', '', ''); + -- ## LOLOR ############################# INSERT INTO projects VALUES ('lolor', 'pge', 4, 0, '', 1, 'https://github.com/pgedge/lolor/tags', 'spock', 1, 'spock.png', 'Logical Replication of Large Objects', 'https://github.com/pgedge/lolor/#spock', 'lola, lolah, kinks'); From 2c9f16f8da2521db20cc6080bbf002e053d7580c Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Mon, 4 Aug 2025 08:16:59 -0500 Subject: [PATCH 37/42] move install url to downloads.pgedge.com (#361) --- cli/scripts/cluster.py | 2 +- cli/scripts/install.py | 2 +- devel/setup/compose/README.md | 6 +++--- devel/setup/compose/proxy_server.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cli/scripts/cluster.py b/cli/scripts/cluster.py index 766abe39..f0395bd5 100755 --- a/cli/scripts/cluster.py +++ b/cli/scripts/cluster.py @@ -15,7 +15,7 @@ BASE_DIR = "cluster" -DEFAULT_REPO = "https://pgedge-download.s3.amazonaws.com/REPO" +DEFAULT_REPO = "https://downloads.pgedge.com/platform/repos/download" def run_cmd( cmd, node, message, verbose, capture_output=False, ignore=False, important=False diff --git a/cli/scripts/install.py b/cli/scripts/install.py index 31225eba..1c402715 100644 --- a/cli/scripts/install.py +++ b/cli/scripts/install.py @@ -4,7 +4,7 @@ import sys, os, tarfile, platform VER = "25.1.0" -REPO = os.getenv("REPO", "https://pgedge-download.s3.amazonaws.com/REPO") +REPO = os.getenv("REPO", "https://downloads.pgedge.com/platform/repos/download") if sys.version_info < (3, 9): maj = sys.version_info.major diff --git a/devel/setup/compose/README.md b/devel/setup/compose/README.md index b21b2e0a..41ae6c30 100644 --- a/devel/setup/compose/README.md +++ b/devel/setup/compose/README.md @@ -63,9 +63,9 @@ If a local package does not exist, it will fallback to a package available in th The chosen repo corresponds to the URL that you choose to configure: -- http://repo:8000/download corresponds with https://pgedge-download.s3.amazonaws.com/REPO -- http://repo:8000/upstream corresponds with https://pgedge-upstream.s3.amazonaws.com/REPO -- http://repo:8000/devel corresponds with https://pgedge-devel.s3.amazonaws.com/REPO +- http://repo:8000/download corresponds with https://downloads.pgedge.com/platform/repos/download +- http://repo:8000/upstream corresponds with https://downloads.pgedge.com/platform/repos/upstream +- http://repo:8000/devel corresponds with https://downloads.pgedge.com/platform/repos/devel The `out` directory within the build container is mounted in the repo container to enable this setup. diff --git a/devel/setup/compose/proxy_server.py b/devel/setup/compose/proxy_server.py index 8a7de274..97b9b858 100644 --- a/devel/setup/compose/proxy_server.py +++ b/devel/setup/compose/proxy_server.py @@ -33,7 +33,7 @@ def do_GET(self): def proxy_request(self, repo, path): """Fetch content from the upstream server and send it to the client.""" - upstream_url = f"https://pgedge-{repo}.s3.amazonaws.com/REPO/{path}" + upstream_url = f"https://downloads.pgedge.com/platform/repos/{repo}/{path}" self.log_message("Proxying request to %s", upstream_url) try: with urllib.request.urlopen(upstream_url) as response: From 669efab142fb5c0841ae8ff9b2d0535b88b7461b Mon Sep 17 00:00:00 2001 From: Muhammad Aqeel Date: Tue, 5 Aug 2025 16:45:58 +0500 Subject: [PATCH 38/42] Bump backrest version to 2.56.0 (#363) --- env.sh | 2 +- src/conf/versions.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/env.sh b/env.sh index 76c3656f..6ee75396 100755 --- a/env.sh +++ b/env.sh @@ -37,7 +37,7 @@ vectorV=0.8.0-1 bouncerV=1.23.1-1 catV=1.2.0 prompgexpV=0.15.0 -backrestV=2.53.1-1 +backrestV=2.56.0-1 wal2jV=2.6.0-1 citusV=12.1.5-1 diff --git a/src/conf/versions.sql b/src/conf/versions.sql index 31f04ab5..2c5c2804 100644 --- a/src/conf/versions.sql +++ b/src/conf/versions.sql @@ -438,8 +438,8 @@ INSERT INTO projects VALUES ('backrest', 'pge', 11, 0, '', 3, 'http://pgbackrest 'backrest', 0, 'backrest.png', 'Backup & Restore', 'http://pgbackrest.org', 'pg_backrest, pgbackrest'); INSERT INTO releases VALUES ('backrest', 2, 'backrest', 'pgBackRest', '', 'test', '', 1, 'MIT', 'EL', ''); -INSERT INTO versions VALUES ('backrest', '2.53.1-1', 'amd, arm', 1, '20240912', '', '', ''); -INSERT INTO versions VALUES ('backrest', '2.53-1', 'amd, arm', 0, '20240729', '', '', ''); +INSERT INTO versions VALUES ('backrest', '2.56.0-1', 'amd, arm', 1, '20240805', '', '', ''); +INSERT INTO versions VALUES ('backrest', '2.53.1-1', 'amd, arm', 0, '20240912', '', '', ''); -- ## PATRONI ########################### INSERT INTO projects VALUES ('patroni', 'app', 11, 0, '', 4, 'https://github.com/pgedge/pgedge-patroni/release', From 89e6251a55f024e58fcade36454567b41b73f637 Mon Sep 17 00:00:00 2001 From: moizpgedge Date: Wed, 30 Jul 2025 19:12:53 +0500 Subject: [PATCH 39/42] escape seq issue solved --- src/pgXX/init-pgXX.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/pgXX/init-pgXX.py b/src/pgXX/init-pgXX.py index 6111c7db..10737e80 100644 --- a/src/pgXX/init-pgXX.py +++ b/src/pgXX/init-pgXX.py @@ -62,7 +62,8 @@ def fatal_error(p_msg): if args.datadir == "": pg_data = os.path.join(data_root, pgver) else: - pg_data = args.datadir + + pg_data = args.datadir.replace(r'\ ', ' ') if not os.path.isdir(pg_data): os.makedirs(pg_data) @@ -195,12 +196,24 @@ def fatal_error(p_msg): util.update_postgresql_conf(pgver, i_port) -if util.get_platform() == "Linux": - os.system("cp " + pgver + "/genSelfCert.sh " + pg_data + "/.") - os.system(pg_data + "/genSelfCert.sh") +# ——— NEW: force Postgres to look in the right place for the certs ——— +conf_file = os.path.join(pg_data, "postgresql.conf") +ssl_block = """ +# — added by init script to enable SSL in data dir (quoting handles spaces) — +ssl = on +ssl_cert_file = '{0}/server.crt' +ssl_key_file = '{0}/server.key' +""".format(pg_data) -os.system("cp " + pgver + "/pg_hba.conf.nix " + pg_data + "/pg_hba.conf") +with open(conf_file, "a") as cf: + cf.write(ssl_block) + +# now generate your cert and copy pg_hba +if util.get_platform() == "Linux": + os.system(f'cp "{pgver}/genSelfCert.sh" "{pg_data}/."') + os.system(f'sh "{pg_data}/genSelfCert.sh"') +os.system(f'cp "{pgver}/pg_hba.conf.nix" "{pg_data}/pg_hba.conf"') if is_password: pg_pass_file = util.remember_pgpassword(pg_password, "*", "*", "*", os_user) else: From d906f1bee7ca4127b1bb294cc8d0c33108624488 Mon Sep 17 00:00:00 2001 From: Moiz Ibrar Date: Wed, 30 Jul 2025 19:49:14 +0500 Subject: [PATCH 40/42] Update init-pgXX.py --- src/pgXX/init-pgXX.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pgXX/init-pgXX.py b/src/pgXX/init-pgXX.py index 10737e80..8d6d25f5 100644 --- a/src/pgXX/init-pgXX.py +++ b/src/pgXX/init-pgXX.py @@ -62,7 +62,6 @@ def fatal_error(p_msg): if args.datadir == "": pg_data = os.path.join(data_root, pgver) else: - pg_data = args.datadir.replace(r'\ ', ' ') if not os.path.isdir(pg_data): From b581a59166c9fc8471c2fe225158b559d3dcb23c Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Tue, 5 Aug 2025 16:15:17 -0500 Subject: [PATCH 41/42] pass datadir as quoted --- cli/scripts/setup.py | 2 +- src/pgXX/init-pgXX.py | 27 ++++++++++----------------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/cli/scripts/setup.py b/cli/scripts/setup.py index 6e5452d8..d5a34632 100755 --- a/cli/scripts/setup.py +++ b/cli/scripts/setup.py @@ -104,7 +104,7 @@ def setup_pgedge(User=None, Passwd=None, dbName=None, port=None, pg_data=None, p util.exit_message( "pg_data cannot be set as relative path. Please specify absolute path instead" ) - pg_init_options = f"--datadir={pg_data}" + pg_init_options = f'--datadir="{pg_data}"' setup_core.check_pre_reqs( User, Passwd, dbName, port, pg_data, pg_major, pg_minor, spock_ver, autostart) diff --git a/src/pgXX/init-pgXX.py b/src/pgXX/init-pgXX.py index 8d6d25f5..9c983d40 100644 --- a/src/pgXX/init-pgXX.py +++ b/src/pgXX/init-pgXX.py @@ -5,6 +5,7 @@ import util, startup import argparse, os, sys, shutil, subprocess, getpass, json +import shlex MY_HOME = os.getenv("MY_HOME", "") @@ -62,7 +63,7 @@ def fatal_error(p_msg): if args.datadir == "": pg_data = os.path.join(data_root, pgver) else: - pg_data = args.datadir.replace(r'\ ', ' ') + pg_data = args.datadir if not os.path.isdir(pg_data): os.makedirs(pg_data) @@ -195,24 +196,16 @@ def fatal_error(p_msg): util.update_postgresql_conf(pgver, i_port) -# ——— NEW: force Postgres to look in the right place for the certs ——— -conf_file = os.path.join(pg_data, "postgresql.conf") -ssl_block = """ -# — added by init script to enable SSL in data dir (quoting handles spaces) — -ssl = on -ssl_cert_file = '{0}/server.crt' -ssl_key_file = '{0}/server.key' -""".format(pg_data) - -with open(conf_file, "a") as cf: - cf.write(ssl_block) - -# now generate your cert and copy pg_hba if util.get_platform() == "Linux": - os.system(f'cp "{pgver}/genSelfCert.sh" "{pg_data}/."') - os.system(f'sh "{pg_data}/genSelfCert.sh"') + gen_cert_src = os.path.join(pgver, "genSelfCert.sh") + gen_cert_dst = os.path.join(pg_data, "genSelfCert.sh") + os.system(f'cp {shlex.quote(gen_cert_src)} {shlex.quote(gen_cert_dst)}') + os.system(f'{shlex.quote(gen_cert_dst)}') + +pg_hba_src = os.path.join(pgver, "pg_hba.conf.nix") +pg_hba_dst = os.path.join(pg_data, "pg_hba.conf") +os.system(f'cp {shlex.quote(pg_hba_src)} {shlex.quote(pg_hba_dst)}') -os.system(f'cp "{pgver}/pg_hba.conf.nix" "{pg_data}/pg_hba.conf"') if is_password: pg_pass_file = util.remember_pgpassword(pg_password, "*", "*", "*", os_user) else: From 97c9e2368eb633c0977e59cdfb70a7c152ee04fa Mon Sep 17 00:00:00 2001 From: Matthew Mols Date: Tue, 5 Aug 2025 21:14:32 -0500 Subject: [PATCH 42/42] bump version to 25.2.0 (#364) --- cli/scripts/install.py | 2 +- cli/scripts/util.py | 2 +- env.sh | 4 ++-- src/conf/versions.sql | 3 ++- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cli/scripts/install.py b/cli/scripts/install.py index 1c402715..42861768 100644 --- a/cli/scripts/install.py +++ b/cli/scripts/install.py @@ -3,7 +3,7 @@ import sys, os, tarfile, platform -VER = "25.1.0" +VER = "25.2.0" REPO = os.getenv("REPO", "https://downloads.pgedge.com/platform/repos/download") if sys.version_info < (3, 9): diff --git a/cli/scripts/util.py b/cli/scripts/util.py index a8dc0ab2..da2cf20e 100644 --- a/cli/scripts/util.py +++ b/cli/scripts/util.py @@ -4,7 +4,7 @@ import os import time -MY_VERSION = "25.1.0" +MY_VERSION = "25.2.0" MY_CODENAME = "" DEFAULT_PG = "17" diff --git a/env.sh b/env.sh index 6ee75396..578e51ef 100755 --- a/env.sh +++ b/env.sh @@ -1,5 +1,5 @@ -hubV=25.1.0 -hubVV=25.1.0 +hubV=25.2.0 +hubVV=25.2.0 aceV=$hubV kirkV=$hubV diff --git a/src/conf/versions.sql b/src/conf/versions.sql index 2c5c2804..b7eb42c8 100644 --- a/src/conf/versions.sql +++ b/src/conf/versions.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS hub; CREATE TABLE hub(v TEXT NOT NULL PRIMARY KEY, c TEXT NOT NULL, d TEXT NOT NULL); -INSERT INTO hub VALUES ('25.1.0', '', '20250626'); +INSERT INTO hub VALUES ('25.2.0', '', '20250815'); DROP VIEW IF EXISTS v_versions; DROP VIEW IF EXISTS v_products; @@ -139,6 +139,7 @@ INSERT INTO projects VALUES ('hub', 'app', 0, 0, 'hub', 0, 'https://github.com/p INSERT INTO releases VALUES ('hub', 1, 'hub', '', '', 'hidden', '', 1, '', '', ''); INSERT INTO versions VALUES ('hub', (select v from hub), '', 1, (select d from hub), '', '', ''); +INSERT INTO versions VALUES ('hub', '25.1.0', '', 0, '20250626', '', '', ''); INSERT INTO versions VALUES ('hub', '25.0.0', '', 0, '20250603', '', '', ''); INSERT INTO versions VALUES ('hub', '24.10.13', '', 0, '20250509', '', '', ''); INSERT INTO versions VALUES ('hub', '24.10.11', '', 0, '20250224', '', '', '');