diff --git a/Helmsman/dsf/external-dsf.yaml b/Helmsman/dsf/external-dsf.yaml index 909657fd..1decc113 100644 --- a/Helmsman/dsf/external-dsf.yaml +++ b/Helmsman/dsf/external-dsf.yaml @@ -114,7 +114,7 @@ apps: databases.mosip_regprc.port: 5433 databases.mosip_pms.enabled: "true" - databases.mosip_pms.branch: "release-1.3.x" + databases.mosip_pms.branch: "v1.3.0-beta.2" databases.mosip_pms.host: "postgres.sandbox.xyz.net" databases.mosip_pms.port: 5433 @@ -129,7 +129,7 @@ apps: databases.mosip_resident.port: 5433 databases.mosip_otp.enabled: "true" - databases.mosip_otp.branch: "release-1.3.x" + databases.mosip_otp.branch: "v1.3.0-beta.1" databases.mosip_otp.host: "postgres.sandbox.xyz.net" databases.mosip_otp.port: 5433 @@ -148,11 +148,11 @@ apps: chart: mosip/keycloak set: image.repository: "mosipid/mosip-artemis-keycloak" - image.tag: "1.2.0.1" + image.tag: "1.2.0.2" image.pullPolicy: "Always" # Override Postgres image postgresql.image.registry: "docker.io" - postgresql.image.repository: "mosipint/postgresql" + postgresql.image.repository: "mosipid/postgresql" postgresql.image.tag: "14.2.0-debian-10-r70" service.type: "ClusterIP" auth.adminUser: "admin" @@ -193,7 +193,7 @@ apps: keycloak-init: namespace: keycloak enabled: true - version: 12.0.2-develop + version: 12.0.2 chart: mosip/keycloak-init set: # image.repository: "mosipid/keycloak-init" @@ -245,7 +245,7 @@ apps: version: 10.1.6 chart: mosip/minio set: - image.repository: "mosipint/minio" + image.repository: "mosipid/minio" image.tag: "2022.2.7-debian-10-r0" metrics.serviceMonitor.enabled: "true" extraEnvVars[0].name: "MINIO_PROMETHEUS_URL" @@ -313,9 +313,9 @@ apps: chart: bitnami/kafka set: # Add these for image overrides - image.repository: "mosipint/kafka" + image.repository: "mosipid/kafka" image.tag: "3.2.1-debian-11-r9" - zookeeper.image.repository: "mosipint/zookeeper" + zookeeper.image.repository: "mosipid/zookeeper" zookeeper.image.tag: "3.8.0-debian-11-r30" clusterDomain: "cluster.local" logRetentionBytes: "_1073741824" diff --git a/Helmsman/dsf/mosip-dsf.yaml b/Helmsman/dsf/mosip-dsf.yaml index 4d20f005..fc9bd37b 100644 --- a/Helmsman/dsf/mosip-dsf.yaml +++ b/Helmsman/dsf/mosip-dsf.yaml @@ -62,14 +62,14 @@ apps: conf-secrets: namespace: conf-secrets enabled: true - version: 12.0.3-develop + version: 12.0.3 chart: mosip/conf-secrets priority: -20 config-server: namespace: config-server enabled: true - version: 12.0.3-develop + version: 12.0.3 chart: mosip/config-server valuesFile: "$WORKDIR/utils/config-server-values.yaml" wait: true @@ -1075,7 +1075,7 @@ apps: mosip-file-server: namespace: mosip-file-server enabled: true - version: 12.0.2-develop + version: 12.0.2 chart: mosip/mosip-file-server set: image.repository: "mosipid/mosip-file-server" diff --git a/Helmsman/dsf/prereq-dsf.yaml b/Helmsman/dsf/prereq-dsf.yaml index 392ad010..dd91d78f 100644 --- a/Helmsman/dsf/prereq-dsf.yaml +++ b/Helmsman/dsf/prereq-dsf.yaml @@ -61,7 +61,7 @@ apps: image.tag: "7.17.2-debian-10-r4" # Kibana configuration and image global.kibanaEnabled: "true" - kibana.image.repository: "mosipint/kibana" + kibana.image.repository: "mosipid/kibana" kibana.image.tag: "7.17.2-debian-10-r0" kibana.image.pullPolicy: IfNotPresent data.heapSize: "728m" @@ -73,7 +73,7 @@ apps: master.persistence.size: "4Gi" master.resources.requests.memory: "728Mi" kibana.persistence.enabled: "false" - sysctlImage.repository: "mosipint/os-shell" + sysctlImage.repository: "mosipid/os-shell" sysctlImage.tag: "12-debian-12-r46" wait: true timeout: 900 diff --git a/Helmsman/dsf/testrigs-dsf.yaml b/Helmsman/dsf/testrigs-dsf.yaml index 12d80ad4..a63ac69e 100644 --- a/Helmsman/dsf/testrigs-dsf.yaml +++ b/Helmsman/dsf/testrigs-dsf.yaml @@ -45,7 +45,7 @@ apps: apitestrig: namespace: apitestrig enabled: true - version: 0.0.1-develop + version: 1.3.4 chart: mosip/apitestrig set: crontime: "0 2 * * *" diff --git a/Helmsman/utils/ansible/README.md b/Helmsman/utils/ansible/README.md new file mode 100644 index 00000000..2d92d8dc --- /dev/null +++ b/Helmsman/utils/ansible/README.md @@ -0,0 +1,157 @@ +# PostgreSQL Secure Setup - MOSIP Infrastructure + +Production-grade PostgreSQL deployment with secure password generation, Kubernetes integration, and private subnet optimization. + +## Quick Start (3 Simple Steps) + +```bash +# 1. Interactive inventory setup with smart defaults +./setup-vm-inventory.sh + +# 2. Deploy PostgreSQL with secure setup +./run-postgresql-playbook.sh + +# 3. Verify installation status +./check-postgresql-status.sh +``` + +## Core Features + +### Deployment Features +- **Streamlined deployment** for private networks +- **Custom port configuration** (default 5433) +- **XFS filesystem** with optimized mount options +- **Connection pooling ready** with high max_connections + +### Production Ready +- **Automated password generation** - no manual password handling +- **Kubernetes integration** - generates secrets and ConfigMaps +- **Data migration support** - moves existing data to new storage +- **Complete lifecycle management** - setup, monitoring, cleanup + +## Essential Scripts + +### Production Scripts +- **`run-postgresql-playbook.sh`** - Main secure deployment script +- **`postgresql-setup.yml`** - Secure Ansible playbook +- **`check-postgresql-status.sh`** - Status monitoring and verification +- **`cleanup-postgresql.sh`** - Safe cleanup with backups + +### Configuration Files +- **`hosts.ini`** - Ansible inventory configuration +- **`postgresql-cleanup.yml`** - Cleanup playbook + +## Configuration + +Edit `hosts.ini` to configure your PostgreSQL server: + +```ini +[postgresql_servers] +postgres-vm ansible_host=10.0.2.176 ansible_user=mosipuser ansible_ssh_private_key_file=~/.ssh/mosip-key + +# Configuration Comments: +# PostgreSQL Version: 15 +# PostgreSQL Port: 5433 +# Storage Device: /dev/nvme2n1 +# Mount Point: /srv/postgres +# Network CIDR: 10.0.0.0/8 +# Kubernetes Namespace: postgres +``` + +## Advanced Usage + +### Security Configuration +```bash +# Use custom Kubernetes namespace and secret name +./run-postgresql-playbook.sh --namespace production --secret-name postgres-creds + +# Auto-confirm deployment (for automation) +./run-postgresql-playbook.sh --auto-confirm +``` + +### Kubernetes Integration +After deployment, Kubernetes files are generated in `/tmp/postgresql-secrets/`: +- `postgres-postgresql.yml` - Secret with credentials +- `postgres-setup-config.yml` - ConfigMap with connection details +- `DEPLOYMENT_INSTRUCTIONS.md` - Deployment guide + +**Apply the generated secrets to Kubernetes:** +```bash +# Create the postgres namespace +kubectl create ns postgres + +# Apply the generated YAML files +kubectl apply -f /tmp/postgresql-secrets/postgres-postgresql.yml +kubectl apply -f /tmp/postgresql-secrets/postgres-setup-config.yml +``` + +### Monitoring and Maintenance +```bash +# Check PostgreSQL status +./check-postgresql-status.sh + +# Safe cleanup (creates backups) +./cleanup-postgresql.sh --safe +``` + +## Security Features + +### Applied Security Measures +- 16-character secure passwords (mixed case, numbers, special chars) +- MD5 password encryption +- Private subnet deployment (no encryption overhead) +- Connection and statement audit logging +- Kubernetes secrets with proper base64 encoding +- Proper file permissions (0600 for secrets) +- Separation of sensitive and non-sensitive data +- No plaintext passwords in logs or files + +## Requirements Checklist + +### Server Requirements + +- [ ] Ubuntu 20.04+ server +- [ ] 4GB+ RAM (8GB+ recommended) +- [ ] Dedicated storage device (e.g., `/dev/nvme2n1`) +- [ ] SSH key-based access configured + +### Network Requirements + +- [ ] Private subnet (10.0.0.0/8 recommended) +- [ ] SSH access on port 22 +- [ ] PostgreSQL port 5433 accessible +- [ ] No encryption requirements (private network) + +### Software Dependencies + +- [ ] Ansible installed +- [ ] Python3-bcrypt package +- [ ] Python3-psycopg2 package +- [ ] SSH keys configured for target server + +## Workflow Overview + +``` +Edit hosts.ini → run-postgresql-playbook.sh → PostgreSQL Ready + ↓ + Kubernetes files generated + ↓ + check-postgresql-status.sh (verify) +``` + +## What Changed from Previous Version + +### Security Enhancements +- **Encryption Optimized**: Streamlined for private subnet deployment +- **Secure Password Generation**: Automated 16-character passwords +- **MD5 Encryption**: Standard password encryption +- **Kubernetes Integration**: Auto-generated secrets and ConfigMaps + +### Operational Excellence +- **Automated Deployment**: One-command secure setup +- **Production Ready**: Enterprise-grade security and monitoring +- **Easy Integration**: Kubernetes-ready with generated manifests + +--- + +*Secure, fast, and production-ready PostgreSQL for MOSIP infrastructure* diff --git a/Helmsman/utils/ansible/check-postgresql-status.sh b/Helmsman/utils/ansible/check-postgresql-status.sh new file mode 100755 index 00000000..566ac138 --- /dev/null +++ b/Helmsman/utils/ansible/check-postgresql-status.sh @@ -0,0 +1,128 @@ +#!/bin/bash + +# PostgreSQL Environment Status Check +# This script checks the current state of PostgreSQL installation before cleanup + +set -e + +INVENTORY_FILE="${1:-hosts.ini}" +TARGET_GROUP="${2:-postgresql_servers}" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +echo -e "${BLUE}=== PostgreSQL Environment Status Check ===${NC}" +echo "Inventory: $INVENTORY_FILE" +echo "Target Group: $TARGET_GROUP" +echo "==============================================" + +# Check connectivity +echo -e "${GREEN}Testing connectivity...${NC}" +if ! ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m ping > /dev/null 2>&1; then + echo -e "${RED}✗ Cannot connect to target hosts${NC}" + exit 1 +fi +echo -e "${GREEN}✓ Connectivity OK${NC}" + +echo "" +echo -e "${YELLOW}=== PostgreSQL Installation Status ===${NC}" + +# Check PostgreSQL packages +echo "PostgreSQL Packages:" +ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a 'dpkg -l | grep postgresql' 2>/dev/null || echo "No PostgreSQL packages found" + +echo "" +echo "PostgreSQL Processes:" +ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a 'ps aux | grep postgres | grep -v grep' 2>/dev/null || echo "No PostgreSQL processes running" + +echo "" +echo "PostgreSQL Service Status:" +ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a 'systemctl is-active postgresql' 2>/dev/null || echo "PostgreSQL service not active" + +echo "" +echo -e "${YELLOW}=== Storage and Mount Points ===${NC}" + +# Check common mount points +for mount_point in "/srv/postgres" "/data" "/mnt/postgresql" "/opt/postgresql"; do + echo "Mount point $mount_point:" + ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a "mount | grep '$mount_point'" 2>/dev/null || echo "Not mounted" +done + +echo "" +echo "Current mount points:" +ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a 'mount | grep -E "(srv|data|postgresql)" || echo "No relevant mount points"' 2>/dev/null + +echo "" +echo -e "${YELLOW}=== Storage Devices ===${NC}" + +# Check common storage devices +for device in "/dev/nvme2n1" "/dev/sdb" "/dev/vdb" "/dev/xvdb"; do + echo "Device $device:" + ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a "lsblk '$device' 2>/dev/null || echo 'Device not found'" 2>/dev/null +done + +echo "" +echo -e "${YELLOW}=== Configuration Files ===${NC}" + +# Check configuration directories +for config_dir in "/etc/postgresql" "/var/lib/postgresql"; do + echo "Directory $config_dir:" + ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a "ls -la '$config_dir' 2>/dev/null || echo 'Directory not found'" 2>/dev/null +done + +echo "" +echo -e "${YELLOW}=== Data Directories ===${NC}" + +# Check data directories +for data_dir in "/srv/postgres/postgresql" "/data/postgresql" "/var/lib/postgresql" "/mnt/postgresql"; do + echo "Data directory $data_dir:" + ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a "ls -la '$data_dir' 2>/dev/null || echo 'Directory not found'" 2>/dev/null +done + +echo "" +echo -e "${YELLOW}=== Network Configuration ===${NC}" + +echo "PostgreSQL listening ports:" +ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a 'netstat -tlnp | grep postgres || echo "No PostgreSQL ports listening"' 2>/dev/null + +echo "" +echo -e "${YELLOW}=== Users and Groups ===${NC}" + +echo "PostgreSQL user:" +ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a 'id postgres 2>/dev/null || echo "User postgres not found"' 2>/dev/null + +echo "" +echo -e "${GREEN}=== Status Summary ===${NC}" + +# Create a summary +ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m shell -a ' +echo "=== PostgreSQL Environment Summary ===" +echo "Hostname: $(hostname)" +echo "Date: $(date)" +echo "" +echo "PostgreSQL packages: $(dpkg -l | grep postgresql | wc -l) found" +echo "PostgreSQL processes: $(ps aux | grep postgres | grep -v grep | wc -l) running" +echo "PostgreSQL config dirs: $(find /etc -name "*postgresql*" -type d 2>/dev/null | wc -l) found" +echo "PostgreSQL data dirs: $(find /var/lib -name "*postgresql*" -type d 2>/dev/null | wc -l) found" +echo "Relevant mount points: $(mount | grep -E "(srv|data|postgresql)" | wc -l) found" +echo "" +if systemctl is-active postgresql >/dev/null 2>&1; then + echo "PostgreSQL service: ACTIVE" +else + echo "PostgreSQL service: INACTIVE" +fi +' 2>/dev/null + +echo "" +echo -e "${BLUE}=== Cleanup Options ===${NC}" +echo "Based on the current state, you can use:" +echo "" +echo "• Quick cleanup (no backups): ./cleanup-postgresql.sh --quick" +echo "• Safe cleanup (with backups): ./cleanup-postgresql.sh --safe" +echo "• Complete cleanup (wipe all): ./cleanup-postgresql.sh --complete" +echo "" +echo "For custom cleanup options, see: ./cleanup-postgresql.sh --help" diff --git a/Helmsman/utils/ansible/cleanup-postgresql.sh b/Helmsman/utils/ansible/cleanup-postgresql.sh new file mode 100755 index 00000000..7e56a2a4 --- /dev/null +++ b/Helmsman/utils/ansible/cleanup-postgresql.sh @@ -0,0 +1,246 @@ +#!/bin/bash + +# PostgreSQL Cleanup Script - Complete Environment Restoration +# This script provides various cleanup options for PostgreSQL installations + +set -e + +# Default values +INVENTORY_FILE="hosts.ini" +CLEANUP_PLAYBOOK="postgresql-cleanup.yml" +TARGET_GROUP="postgresql_servers" +POSTGRESQL_VERSION="15" +STORAGE_DEVICE="/dev/nvme2n1" +MOUNT_POINT="/srv/postgres" +CREATE_BACKUP=false +WIPE_DEVICE=false +AUTO_CONFIRM=false + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +print_usage() { + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " -i, --inventory FILE Inventory file (default: hosts.ini)" + echo " -c, --cleanup FILE Cleanup playbook (default: postgresql-cleanup.yml)" + echo " -v, --version VERSION PostgreSQL version (default: 15)" + echo " -d, --device DEVICE Storage device (default: /dev/nvme2n1)" + echo " -m, --mount PATH Mount point (default: /srv/postgres)" + echo " -g, --group GROUP Target group (default: postgresql_servers)" + echo " -b, --backup Create safety backup before cleanup" + echo " -w, --wipe-device Wipe storage device completely (DANGEROUS!)" + echo " -y, --yes Auto-confirm (skip prompts)" + echo " --quick Quick cleanup (no backups, no device wipe)" + echo " --safe Safe cleanup (with backups, no device wipe)" + echo " --complete Complete cleanup (with backups and device wipe)" + echo " -t, --test Test connectivity only" + echo " -h, --help Show this help message" + echo "" + echo "Cleanup Modes:" + echo " Quick: Fast cleanup, no backups, preserves device data" + echo " Safe: Creates backups, removes PostgreSQL, preserves device" + echo " Complete: Creates backups, removes PostgreSQL, wipes device clean" + echo "" + echo "Examples:" + echo " $0 --safe # Safe cleanup with backups" + echo " $0 --quick -y # Quick cleanup, auto-confirm" + echo " $0 --complete # Complete cleanup with device wipe" + echo " $0 -b -d /dev/vdb -m /opt/postgresql # Custom device and mount with backup" + echo " $0 -t # Test connectivity only" +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -i|--inventory) + INVENTORY_FILE="$2" + shift 2 + ;; + -c|--cleanup) + CLEANUP_PLAYBOOK="$2" + shift 2 + ;; + -v|--version) + POSTGRESQL_VERSION="$2" + shift 2 + ;; + -d|--device) + STORAGE_DEVICE="$2" + shift 2 + ;; + -m|--mount) + MOUNT_POINT="$2" + shift 2 + ;; + -g|--group) + TARGET_GROUP="$2" + shift 2 + ;; + -b|--backup) + CREATE_BACKUP=true + shift + ;; + -w|--wipe-device) + WIPE_DEVICE=true + shift + ;; + -y|--yes) + AUTO_CONFIRM=true + shift + ;; + --quick) + CREATE_BACKUP=false + WIPE_DEVICE=false + shift + ;; + --safe) + CREATE_BACKUP=true + WIPE_DEVICE=false + shift + ;; + --complete) + CREATE_BACKUP=true + WIPE_DEVICE=true + shift + ;; + -t|--test) + TEST_ONLY=true + shift + ;; + -h|--help) + print_usage + exit 0 + ;; + *) + echo -e "${RED}Error: Unknown option $1${NC}" + print_usage + exit 1 + ;; + esac +done + +echo -e "${RED}=== PostgreSQL Cleanup Script ===${NC}" +echo -e "${YELLOW}WARNING: This will PERMANENTLY DELETE PostgreSQL and its data!${NC}" +echo "" +echo "Configuration:" +echo " Inventory: $INVENTORY_FILE" +echo " Playbook: $CLEANUP_PLAYBOOK" +echo " Target Group: $TARGET_GROUP" +echo " PostgreSQL Version: $POSTGRESQL_VERSION" +echo " Storage Device: $STORAGE_DEVICE" +echo " Mount Point: $MOUNT_POINT" +echo " Create Backup: $CREATE_BACKUP" +echo " Wipe Device: $WIPE_DEVICE" +echo " Auto Confirm: $AUTO_CONFIRM" +echo "==============================================" + +# Check if required files exist +if [[ ! -f "$INVENTORY_FILE" ]]; then + echo -e "${RED}Error: Inventory file '$INVENTORY_FILE' not found!${NC}" + exit 1 +fi + +if [[ ! -f "$CLEANUP_PLAYBOOK" ]]; then + echo -e "${RED}Error: Cleanup playbook '$CLEANUP_PLAYBOOK' not found!${NC}" + exit 1 +fi + +# Check if Ansible is installed +if ! command -v ansible &> /dev/null; then + echo -e "${RED}Error: Ansible not found. Please install Ansible first.${NC}" + exit 1 +fi + +# Test connectivity +echo -e "${GREEN}Testing connectivity...${NC}" +if ansible "$TARGET_GROUP" -i "$INVENTORY_FILE" -m ping; then + echo -e "${GREEN}✓ Connectivity test passed${NC}" +else + echo -e "${RED}✗ Connectivity test failed${NC}" + echo "Please check your inventory file and SSH configuration" + exit 1 +fi + +# If test only, exit here +if [[ "${TEST_ONLY:-false}" == "true" ]]; then + echo -e "${GREEN}Test completed successfully!${NC}" + exit 0 +fi + +# Show what will be cleaned up +echo -e "${YELLOW}=== Cleanup Preview ===${NC}" +echo "The following will be PERMANENTLY DELETED:" +echo "• PostgreSQL $POSTGRESQL_VERSION service and packages" +echo "• All PostgreSQL configuration files" +echo "• All PostgreSQL data directories" +echo "• Mount point $MOUNT_POINT" +if [[ "$WIPE_DEVICE" == "true" ]]; then + echo -e "${RED}• ALL DATA on storage device $STORAGE_DEVICE (COMPLETE WIPE!)${NC}" +fi +echo "" + +if [[ "$CREATE_BACKUP" == "true" ]]; then + echo -e "${GREEN}Safety measures enabled:${NC}" + echo "• Configuration files will be backed up" + echo "• Data will be backed up before deletion" +fi +echo "" + +# Final confirmation +if [[ "$AUTO_CONFIRM" != "true" ]]; then + if [[ "$WIPE_DEVICE" == "true" ]]; then + echo -e "${RED}⚠️ DANGER: Device wipe is enabled! This will destroy ALL data on $STORAGE_DEVICE${NC}" + echo -e "${RED}⚠️ This action is IRREVERSIBLE!${NC}" + read -p "Type 'DELETE EVERYTHING' to confirm device wipe: " device_confirm + if [[ "$device_confirm" != "DELETE EVERYTHING" ]]; then + echo "Device wipe cancelled." + exit 1 + fi + fi + + read -p "Are you sure you want to proceed with PostgreSQL cleanup? (y/N): " confirm + if [[ $confirm != [yY] && $confirm != [yY][eE][sS] ]]; then + echo "Cleanup cancelled." + exit 0 + fi +fi + +# Show the actual ansible-playbook command +echo -e "${YELLOW}Executing cleanup command:${NC}" +echo "ansible-playbook -i $INVENTORY_FILE $CLEANUP_PLAYBOOK \\" +echo " --limit $TARGET_GROUP \\" +echo " --extra-vars \"postgresql_version=$POSTGRESQL_VERSION\" \\" +echo " --extra-vars \"storage_device=$STORAGE_DEVICE\" \\" +echo " --extra-vars \"mount_point=$MOUNT_POINT\" \\" +echo " --extra-vars \"create_backup=$CREATE_BACKUP\" \\" +echo " --extra-vars \"wipe_device=$WIPE_DEVICE\" \\" +echo " --extra-vars \"auto_confirm=$AUTO_CONFIRM\" \\" +echo " -v" +echo "" + +# Execute the cleanup +echo -e "${RED}Starting PostgreSQL cleanup...${NC}" +ansible-playbook -i "$INVENTORY_FILE" "$CLEANUP_PLAYBOOK" \ + --limit "$TARGET_GROUP" \ + --extra-vars "postgresql_version=$POSTGRESQL_VERSION" \ + --extra-vars "storage_device=$STORAGE_DEVICE" \ + --extra-vars "mount_point=$MOUNT_POINT" \ + --extra-vars "create_backup=$CREATE_BACKUP" \ + --extra-vars "wipe_device=$WIPE_DEVICE" \ + --extra-vars "auto_confirm=$AUTO_CONFIRM" \ + -v + +echo -e "${GREEN}PostgreSQL cleanup completed successfully!${NC}" +echo "" +echo -e "${BLUE}=== Post-Cleanup Verification ===${NC}" +echo "You can verify the cleanup with these commands:" +echo "ansible $TARGET_GROUP -i $INVENTORY_FILE -m shell -a 'dpkg -l | grep postgresql'" +echo "ansible $TARGET_GROUP -i $INVENTORY_FILE -m shell -a 'ps aux | grep postgres'" +echo "ansible $TARGET_GROUP -i $INVENTORY_FILE -m shell -a 'mount | grep $MOUNT_POINT'" +echo "ansible $TARGET_GROUP -i $INVENTORY_FILE -m shell -a 'lsblk $STORAGE_DEVICE'" diff --git a/Helmsman/utils/ansible/hosts.ini b/Helmsman/utils/ansible/hosts.ini new file mode 100644 index 00000000..a4d206e0 --- /dev/null +++ b/Helmsman/utils/ansible/hosts.ini @@ -0,0 +1,34 @@ +# Ansible Inventory for PostgreSQL Setup on Virtual Machine +# Generated by setup-vm-inventory.sh on Tue 26 Aug 2025 12:35:42 AM IST +# +# Configuration Summary: +# - PostgreSQL Version: 15 +# - Storage Device: /dev/nvme2n1 +# - Mount Point: /srv/postgres +# - Network CIDR: 10.0.0.0/8 +# - PostgreSQL Port: 5433 + +[postgresql_servers] +vm_server ansible_host= ansible_user=ubuntu ansible_ssh_private_key_file=/path/sandbox.pem ansible_port=22 + +[postgresql_servers:vars] +# SSH and Ansible Configuration +ansible_become=yes +ansible_become_method=sudo +ansible_ssh_common_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' +ansible_python_interpreter=/usr/bin/python3 +ansible_ssh_timeout=30 +ansible_connect_timeout=30 + +# PostgreSQL Configuration Variables +postgresql_version=15 +storage_device=/dev/nvme2n1 +mount_point=/srv/postgres +network_cidr=10.0.0.0/8 + +# Additional PostgreSQL Variables +postgresql_port=5433 + +# For local testing (if needed) +[local] +localhost ansible_connection=local diff --git a/Helmsman/utils/ansible/postgresql-cleanup.yml b/Helmsman/utils/ansible/postgresql-cleanup.yml new file mode 100644 index 00000000..dd16e34b --- /dev/null +++ b/Helmsman/utils/ansible/postgresql-cleanup.yml @@ -0,0 +1,232 @@ +--- +# PostgreSQL Cleanup - Complete Removal and Restoration +# This playbook completely removes PostgreSQL and restores the system to original state + +- name: PostgreSQL Complete Cleanup + hosts: postgresql_servers + become: yes + vars: + # Configuration - should match the setup playbook + postgres_version: "{{ postgresql_version | default('15') }}" + postgres_port: "{{ postgresql_port | default('5433') }}" + postgres_device: "{{ storage_device | default('/dev/nvme2n1') }}" + postgres_mount: "{{ mount_point | default('/srv/postgres') }}" + postgres_data_dir: "{{ postgres_mount }}/postgresql/{{ postgres_version }}/main" + postgres_old_dir: "/var/lib/postgresql/{{ postgres_version }}/main" + postgres_config_dir: "/etc/postgresql/{{ postgres_version }}/main" + + tasks: + - name: Display cleanup summary + debug: + msg: | + PostgreSQL {{ postgres_version }} Cleanup Starting + Port: {{ postgres_port }} + Data Directory: {{ postgres_data_dir }} + Mount Point: {{ postgres_mount }} + Device: {{ postgres_device }} + WARNING: This will PERMANENTLY DELETE all PostgreSQL data! + + - name: Pause for confirmation + pause: + prompt: "Are you sure you want to proceed with cleanup? This will DELETE ALL PostgreSQL data! Press ENTER to continue or Ctrl+C to abort" + when: not (auto_confirm | default(false) | bool) + + # Stop PostgreSQL Service + - name: Stop PostgreSQL service + systemd: + name: postgresql + state: stopped + ignore_errors: yes + + - name: Disable PostgreSQL service + systemd: + name: postgresql + enabled: no + ignore_errors: yes + + # Backup current data (optional safety measure) + - name: Create backup directory (safety measure) + file: + path: "/tmp/postgresql-cleanup-backup-{{ ansible_date_time.epoch }}" + state: directory + register: backup_dir + when: create_backup | default(false) | bool + + - name: Backup PostgreSQL configuration files + copy: + src: "{{ postgres_config_dir }}/" + dest: "{{ backup_dir.path }}/config/" + remote_src: yes + when: create_backup | default(false) | bool and backup_dir is defined + ignore_errors: yes + + - name: Backup custom data directory + shell: | + if [ -d "{{ postgres_data_dir }}" ]; then + tar -czf "{{ backup_dir.path }}/data-backup.tar.gz" -C "{{ postgres_data_dir }}" . + fi + when: create_backup | default(false) | bool and backup_dir is defined + ignore_errors: yes + + # Remove PostgreSQL completely + - name: Remove PostgreSQL packages + apt: + name: + - postgresql-{{ postgres_version }} + - postgresql-client-{{ postgres_version }} + - postgresql-contrib-{{ postgres_version }} + state: absent + purge: yes + autoremove: yes + + - name: Remove PostgreSQL common packages + apt: + name: + - postgresql-common + - postgresql-client-common + state: absent + purge: yes + autoremove: yes + ignore_errors: yes + + # Remove PostgreSQL APT repository + - name: Remove PostgreSQL APT repository key + apt_key: + url: https://www.postgresql.org/media/keys/ACCC4CF8.asc + state: absent + ignore_errors: yes + + - name: Remove PostgreSQL APT repository + apt_repository: + repo: "deb http://apt.postgresql.org/pub/repos/apt {{ ansible_distribution_release }}-pgdg main" + state: absent + ignore_errors: yes + + # Clean up directories and files + - name: Remove PostgreSQL configuration directory + file: + path: /etc/postgresql + state: absent + + - name: Remove PostgreSQL log directory + file: + path: /var/log/postgresql + state: absent + + - name: Remove PostgreSQL lib directory + file: + path: /var/lib/postgresql + state: absent + + - name: Remove custom data directory + file: + path: "{{ postgres_data_dir }}" + state: absent + when: postgres_data_dir != "/var/lib/postgresql/{{ postgres_version }}/main" + + # Unmount and clean storage + - name: Check if mount point is mounted + shell: "mount | grep '{{ postgres_mount }}'" + register: mount_check + failed_when: false + changed_when: false + + - name: Unmount the storage device + mount: + path: "{{ postgres_mount }}" + state: unmounted + when: mount_check.rc == 0 + + - name: Remove mount entry from fstab + mount: + path: "{{ postgres_mount }}" + state: absent + + - name: Remove mount directory + file: + path: "{{ postgres_mount }}" + state: absent + + # Optional: Wipe the storage device (DANGEROUS!) + - name: Wipe storage device (if requested) + shell: | + if [ -b "{{ postgres_device }}" ]; then + wipefs -a "{{ postgres_device }}" + dd if=/dev/zero of="{{ postgres_device }}" bs=1M count=100 + fi + when: wipe_device | default(false) | bool + ignore_errors: yes + + # Remove postgres user and group + - name: Remove postgres user + user: + name: postgres + state: absent + remove: yes + ignore_errors: yes + + - name: Remove postgres group + group: + name: postgres + state: absent + ignore_errors: yes + + # Clean up any remaining packages + - name: Remove orphaned packages + shell: apt autoremove -y + ignore_errors: yes + + - name: Clean package cache + shell: apt autoclean + ignore_errors: yes + + # Restore original pg_hba.conf backup if exists + - name: Check for original pg_hba.conf backup + stat: + path: "{{ postgres_config_dir }}/pg_hba.conf.*~" + register: original_backup + ignore_errors: yes + + # Final verification + - name: Verify PostgreSQL removal + shell: | + echo "=== PostgreSQL Cleanup Verification ===" + echo "PostgreSQL packages:" + dpkg -l | grep postgresql || echo "No PostgreSQL packages found" + echo "" + echo "PostgreSQL processes:" + ps aux | grep postgres | grep -v grep || echo "No PostgreSQL processes running" + echo "" + echo "Mount points:" + mount | grep "{{ postgres_mount }}" || echo "No mount points found" + echo "" + echo "Storage device:" + lsblk "{{ postgres_device }}" 2>/dev/null || echo "Device available" + echo "" + echo "=== Cleanup Complete ===" + register: cleanup_verification + changed_when: false + + - name: Display cleanup results + debug: + var: cleanup_verification.stdout_lines + + - name: Display backup information + debug: + msg: | + Cleanup completed successfully! + {% if create_backup | default(false) | bool and backup_dir is defined %} + Safety backup created at: {{ backup_dir.path }} + {% endif %} + + System Status: + - PostgreSQL service: Stopped and disabled + - PostgreSQL packages: Removed + - Configuration files: Deleted + - Data directories: Deleted + - Mount points: Unmounted and removed + {% if wipe_device | default(false) | bool %} + - Storage device: Wiped clean + {% endif %} + + The system has been restored to its original state. diff --git a/Helmsman/utils/ansible/postgresql-setup.yml b/Helmsman/utils/ansible/postgresql-setup.yml new file mode 100644 index 00000000..d418520d --- /dev/null +++ b/Helmsman/utils/ansible/postgresql-setup.yml @@ -0,0 +1,591 @@ +--- +# PostgreSQL Setup with Secure Password Generation - Production Grade +# Enhanced playbook with secure password generation and Kubernetes secret creation + +- name: PostgreSQL Secure Setup + hosts: postgresql_servers + become: yes + vars: + # Default configuration - easily overridable + postgres_version: "{{ postgresql_version | default('15') }}" + postgres_port: "{{ postgresql_port | default('5433') }}" + postgres_device: "{{ storage_device | default('/dev/nvme2n1') }}" + postgres_mount: "{{ mount_point | default('/srv/postgres') }}" + postgres_data_dir: "{{ postgres_mount }}/postgresql/{{ postgres_version }}/main" + postgres_old_dir: "{% if ansible_os_family == 'Debian' %}/var/lib/postgresql/{{ postgres_version }}/main{% else %}/var/lib/pgsql/{{ postgres_version }}/data{% endif %}" + postgres_config_dir: "{% if ansible_os_family == 'Debian' %}/etc/postgresql/{{ postgres_version }}/main{% else %}/var/lib/pgsql/{{ postgres_version }}/data{% endif %}" + postgres_bin_dir: "{% if ansible_os_family == 'Debian' %}/usr/lib/postgresql/{{ postgres_version }}/bin{% else %}/usr/pgsql-{{ postgres_version }}/bin{% endif %}" + postgres_service_name: "{% if ansible_os_family == 'Debian' %}postgresql{% else %}postgresql-{{ postgres_version }}{% endif %}" + postgres_network_cidr: "{{ network_cidr | default('10.0.0.0/8') }}" + postgres_namespace: "{{ kubernetes_namespace | default('postgres') }}" + postgres_secret_name: "{{ secret_name | default('postgres-postgresql') }}" + # Use localhost for local operations, but actual IP for ConfigMap + postgres_local_host: "{{ local_connection_host | default('localhost') }}" + postgres_external_host: "{{ ansible_host }}" + + tasks: + # OS Compatibility and Prerequisites + - name: Gather OS facts + setup: + gather_subset: + - 'distribution' + - 'distribution_version' + + - name: Ensure Python3 is available (CentOS/RHEL compatibility) + package: + name: python3 + state: present + when: ansible_os_family == "RedHat" + become: yes + + - name: Verify Python3 with secrets module availability + command: python3 -c "import secrets; print('Python secrets module available')" + register: python_secrets_check + failed_when: python_secrets_check.rc != 0 + changed_when: false + + # Prerequisites Check + - name: Check if device exists + stat: + path: "{{ postgres_device }}" + follow: yes + register: device_check + failed_when: not device_check.stat.exists or not device_check.stat.isblk + + # Secure Password Generation - Idempotent approach + - name: Check if PostgreSQL is already configured with a password + shell: | + sudo -u postgres psql -p {{ postgres_port }} -c "SELECT 1;" 2>/dev/null + register: postgres_status_check + failed_when: false + changed_when: false + + - name: Try to read existing encrypted password from stored file + shell: | + if [ -f /var/lib/postgresql/.postgres_password_enc ] && [ -f /var/lib/postgresql/.postgres_encryption_key ]; then + ENCRYPTION_KEY=$(cat /var/lib/postgresql/.postgres_encryption_key) + openssl enc -aes-256-cbc -d -salt -pbkdf2 -pass pass:"$ENCRYPTION_KEY" -in /var/lib/postgresql/.postgres_password_enc 2>/dev/null + else + exit 1 + fi + register: existing_encrypted_password + failed_when: false + changed_when: false + no_log: true + + - name: Set existing password if file exists and is valid + set_fact: + postgres_password: "{{ existing_encrypted_password.stdout }}" + when: + - existing_encrypted_password.rc == 0 + - existing_encrypted_password.stdout is defined + - (existing_encrypted_password.stdout | length) == 16 + no_log: true + + - name: Generate secure PostgreSQL password only if needed (OS-independent method) + shell: | + # Primary Method: Python secrets (works on all OS with Python 3.6+) + python3 -c " + import secrets + import string + # Use cryptographically secure random generator + uppercase = ''.join(secrets.choice(string.ascii_uppercase) for _ in range(4)) + lowercase = ''.join(secrets.choice(string.ascii_lowercase) for _ in range(4)) + digits = ''.join(secrets.choice(string.digits) for _ in range(4)) + special = ''.join(secrets.choice('!@#\$%^&*') for _ in range(4)) + password = list(uppercase + lowercase + digits + special) + secrets.SystemRandom().shuffle(password) + print(''.join(password)) + " + register: generated_password_primary + when: postgres_password is not defined + no_log: true + + - name: Generate secure PostgreSQL password using OpenSSL fallback (Linux/macOS only) + shell: | + # Fallback Method: OpenSSL (Linux/macOS optimized) + openssl rand -base64 24 | tr -d "=+/" | cut -c1-16 + register: generated_password_fallback + when: + - postgres_password is not defined + - generated_password_primary.failed is defined + - generated_password_primary.failed == true + no_log: true + failed_when: false + + - name: Set password from OS-independent generation + set_fact: + postgres_password: "{{ generated_password_primary.stdout if generated_password_primary.stdout is defined else generated_password_fallback.stdout }}" + when: postgres_password is not defined + no_log: true + + - name: Validate password complexity + assert: + that: + - postgres_password | length == 16 + - postgres_password | regex_search('[A-Z]') + - postgres_password | regex_search('[a-z]') + - postgres_password | regex_search('[0-9]') + - postgres_password | regex_search('[!@#$%^&*]') + fail_msg: "Generated password does not meet complexity requirements" + success_msg: "Password meets security requirements" + no_log: true + + # Store password in a fact to ensure consistency throughout playbook + - name: Store password in host facts for consistency + set_fact: + postgres_password: "{{ postgres_password }}" + no_log: true + + # Storage Setup + - name: Check if device is already mounted + command: mountpoint -q {{ postgres_mount }} + register: mount_check + failed_when: false + changed_when: false + + - name: Format device if not mounted + filesystem: + fstype: xfs + dev: "{{ postgres_device }}" + force: no + when: mount_check.rc != 0 + + - name: Create mount point and mount device + mount: + path: "{{ postgres_mount }}" + src: "{{ postgres_device }}" + fstype: xfs + opts: defaults,noatime + state: mounted + + # PostgreSQL Installation - Multi-OS Support + + # Ubuntu/Debian Installation + - name: Install required packages for repository (Ubuntu/Debian) + apt: + name: + - wget + - ca-certificates + - gnupg + - lsb-release + - python3-bcrypt + - python3-psycopg2 + state: present + update_cache: yes + when: ansible_os_family == "Debian" + + # CentOS/RHEL Installation + - name: Install PostgreSQL repository (CentOS/RHEL) + yum: + name: "https://download.postgresql.org/pub/repos/yum/reporpms/EL-{{ ansible_distribution_major_version }}-x86_64/pgdg-redhat-repo-latest.noarch.rpm" + state: present + disable_gpg_check: yes + when: ansible_os_family == "RedHat" + + - name: Install required packages (CentOS/RHEL) + yum: + name: + - postgresql{{ postgres_version }}-server + - postgresql{{ postgres_version }} + - python3-bcrypt + - python3-psycopg2 + - wget + - ca-certificates + state: present + when: ansible_os_family == "RedHat" + + # Ubuntu/Debian Repository Setup + - name: Get Ubuntu codename + command: lsb_release -cs + register: ubuntu_codename + changed_when: false + when: ansible_os_family == "Debian" + + - name: Add PostgreSQL signing key + apt_key: + url: https://www.postgresql.org/media/keys/ACCC4CF8.asc + state: present + when: ansible_os_family == "Debian" + + - name: Add PostgreSQL APT repository + apt_repository: + repo: "deb http://apt.postgresql.org/pub/repos/apt {{ ubuntu_codename.stdout }}-pgdg main" + state: present + when: ansible_os_family == "Debian" + + - name: Update package cache after adding PostgreSQL repo + apt: + update_cache: yes + when: ansible_os_family == "Debian" + + # Install PostgreSQL (Ubuntu/Debian) + - name: Install PostgreSQL packages (Ubuntu/Debian) + apt: + name: + - "postgresql-{{ postgres_version }}" + - "postgresql-client-{{ postgres_version }}" + state: present + when: ansible_os_family == "Debian" + + # Initialize PostgreSQL on CentOS/RHEL (required step) + - name: Initialize PostgreSQL database (CentOS/RHEL only) + command: "/usr/pgsql-{{ postgres_version }}/bin/postgresql-{{ postgres_version }}-setup initdb" + when: + - ansible_os_family == "RedHat" + become: yes + register: postgres_init_result + failed_when: false + changed_when: postgres_init_result.rc == 0 + + # Generate encryption key AFTER PostgreSQL installation (so postgres user and directories exist) + - name: Generate or retrieve encryption key for password storage + shell: | + if [ ! -f /var/lib/postgresql/.postgres_encryption_key ]; then + # Generate a secure random key for encryption + openssl rand -base64 32 > /var/lib/postgresql/.postgres_encryption_key + chmod 600 /var/lib/postgresql/.postgres_encryption_key + chown postgres:postgres /var/lib/postgresql/.postgres_encryption_key + fi + cat /var/lib/postgresql/.postgres_encryption_key + register: encryption_key + no_log: true + + - name: Store password securely for future runs (after PostgreSQL installation) + shell: | + # Create encrypted password file using secure encryption key + echo "{{ postgres_password }}" | openssl enc -aes-256-cbc -salt -pbkdf2 -pass pass:"{{ encryption_key.stdout }}" -out /var/lib/postgresql/.postgres_password_enc + chmod 600 /var/lib/postgresql/.postgres_password_enc + chown postgres:postgres /var/lib/postgresql/.postgres_password_enc + # Also store a hash for verification + echo "{{ postgres_password }}" | sha256sum | cut -d' ' -f1 > /var/lib/postgresql/.postgres_password_hash + chmod 600 /var/lib/postgresql/.postgres_password_hash + chown postgres:postgres /var/lib/postgresql/.postgres_password_hash + when: postgres_password is defined + no_log: true + + - name: Stop PostgreSQL for data migration + systemd: + name: postgresql + state: stopped + + # Data Migration + - name: Check if old data directory exists + stat: + path: "{{ postgres_old_dir }}/PG_VERSION" + register: old_data_exists + + - name: Check if new data directory is empty + find: + paths: "{{ postgres_data_dir }}" + file_type: any + register: new_data_check + failed_when: false + + - name: Create new data directory + file: + path: "{{ postgres_data_dir }}" + state: directory + owner: postgres + group: postgres + mode: '0700' + recurse: yes + + - name: Copy data to new location + shell: | + sudo -u postgres cp -a {{ postgres_old_dir }}/* {{ postgres_data_dir }}/ + sudo -u postgres cp -a {{ postgres_old_dir }}/.[^.]* {{ postgres_data_dir }}/ 2>/dev/null || true + when: + - old_data_exists.stat.exists + - new_data_check.matched == 0 + + - name: Fix permissions on data directory + file: + path: "{{ postgres_data_dir }}" + owner: postgres + group: postgres + mode: '0700' + recurse: yes + + # Configuration + - name: Configure PostgreSQL settings + lineinfile: + path: "{{ postgres_config_dir }}/postgresql.conf" + regexp: "{{ item.regexp }}" + line: "{{ item.line }}" + backup: yes + loop: + - { regexp: '^#?data_directory\s*=', line: "data_directory = '{{ postgres_data_dir }}'" } + - { regexp: '^#?port\s*=', line: "port = {{ postgres_port }}" } + - { regexp: '^#?listen_addresses\s*=', line: "listen_addresses = '*'" } + - { regexp: '^#?max_connections\s*=', line: "max_connections = 1000" } + notify: restart postgresql + + - name: Configure pg_hba.conf for access + copy: + dest: "{{ postgres_config_dir }}/pg_hba.conf" + content: | + # Database administrative login by Unix domain socket + local all postgres peer + + # "local" is for Unix domain socket connections only + local all all peer + + # IPv4 local connections: + host all all 0.0.0.0/0 md5 + + # IPv6 local connections: + host all all ::1/128 md5 + + # Allow replication connections from localhost, by a user with the + # replication privilege. + local replication all peer + host replication all 127.0.0.1/32 md5 + host replication all ::1/128 md5 + owner: postgres + group: postgres + mode: '0640' + backup: yes + notify: restart postgresql + + # Start Service + - name: Start and enable PostgreSQL + systemd: + name: postgresql + state: started + enabled: yes + + # Wait for PostgreSQL to be ready + - name: Wait for PostgreSQL to be ready + wait_for: + port: "{{ postgres_port }}" + timeout: 30 + + # Secure Password Setup - Only set if not already configured + - name: Test if password is already set correctly + shell: | + export PGPASSWORD='{{ postgres_password }}' + psql -h localhost -p {{ postgres_port }} -U postgres -c "SELECT 'Password already configured' as status;" + register: existing_password_test + no_log: true + failed_when: false + changed_when: false + + - name: Set PostgreSQL postgres user password only if needed + shell: | + export PGPASSWORD_NEW='{{ postgres_password }}' + sudo -u postgres psql -p {{ postgres_port }} </dev/null || true + find /var/tmp -name "*postgres*" -type f -mmin -5 ! -name "*.yml" -delete 2>/dev/null || true + # Clear any environment variables that might contain passwords + unset PGPASSWORD PGPASSWORD_NEW 2>/dev/null || true + changed_when: false + failed_when: false + + # Clear sensitive variables from memory + - name: Clear sensitive variables from Ansible facts + set_fact: + postgres_password: "" + postgres_password_b64: "" + generated_password: "" + generated_password_primary: "" + generated_password_fallback: "" + no_log: true + + # Verify Kubernetes files were created + - name: Verify Kubernetes files were created successfully on localhost + stat: + path: "{{ item }}" + register: k8s_files_check + loop: + - /tmp/postgresql-secrets/{{ postgres_secret_name }}.yml + - /tmp/postgresql-secrets/postgres-setup-config.yml + delegate_to: localhost + run_once: true + become: false + when: postgres_password_b64 is defined + + - name: Display Kubernetes files status + debug: + msg: | + Kubernetes Files Status: + {% for file in k8s_files_check.results %} + - {{ file.item }}: {{ 'EXISTS' if file.stat.exists else 'MISSING' }} + {% endfor %} + when: k8s_files_check is defined + + # Final verification and summary + - name: Display setup summary + debug: + msg: | + ✅ PostgreSQL {{ postgres_version }} Secure Setup Complete + + Configuration: + - Host: {{ ansible_host }} + - Port: {{ postgres_port }} + - Data Directory: {{ postgres_data_dir }} + - Mount Point: {{ postgres_mount }} + - Password: {{ 'REUSED EXISTING' if existing_password_test.rc == 0 else 'NEWLY GENERATED' }} + - Password Storage: /var/lib/postgresql/.postgres_password_enc (AES-256-CBC encrypted) + - Connection Test: {{ 'PASSED' if pg_test is succeeded else 'FAILED' }} + + Kubernetes Files Generated: + - Secret: /tmp/postgresql-secrets/{{ postgres_secret_name }}.yml + - ConfigMap: /tmp/postgresql-secrets/postgres-setup-config.yml + + Security Features: + - 16-character secure password (persistent across runs) + - AES-256-CBC encrypted password storage with PBKDF2 + - Cryptographically secure encryption key (randomly generated) + - MD5 authentication for database connections + - No plaintext passwords in logs or process lists + - Environment variable isolation + - Secure memory cleanup + - Private subnet deployment + - Idempotent execution + - Kubernetes secrets with proper encoding + + Next Steps: + 1. Review files in /tmp/postgresql-secrets/ + 2. Deploy to Kubernetes cluster + 3. Password will be reused on subsequent playbook runs + + Note: Password is securely encrypted with a randomly generated key stored at: + /var/lib/postgresql/.postgres_encryption_key + + handlers: + - name: restart postgresql + systemd: + name: postgresql + state: restarted diff --git a/Helmsman/utils/ansible/run-postgresql-playbook.sh b/Helmsman/utils/ansible/run-postgresql-playbook.sh new file mode 100755 index 00000000..22046eda --- /dev/null +++ b/Helmsman/utils/ansible/run-postgresql-playbook.sh @@ -0,0 +1,260 @@ +#!/bin/bash + +# PostgreSQL Secure Setup Script +# Production-grade deployment with secure password generation and Kubernetes integration + +set -euo pipefail + +# Configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +INVENTORY_FILE="${SCRIPT_DIR}/hosts.ini" +PLAYBOOK_FILE="${SCRIPT_DIR}/postgresql-setup.yml" +SECRETS_DIR="/tmp/postgresql-secrets" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Logging function +log() { + echo -e "${BLUE}[$(date +'%Y-%m-%d %H:%M:%S')]${NC} $1" +} + +error() { + echo -e "${RED}[ERROR]${NC} $1" >&2 +} + +success() { + echo -e "${GREEN}[SUCCESS]${NC} $1" +} + +warning() { + echo -e "${YELLOW}[WARNING]${NC} $1" +} + +# Print banner +print_banner() { + cat << 'EOF' +╔══════════════════════════════════════════════════════════════╗ +║ PostgreSQL Secure Setup ║ +║ Production-Grade with Kubernetes Integration ║ +╚══════════════════════════════════════════════════════════════╝ +EOF +} + +# Check prerequisites +check_prerequisites() { + log "Checking prerequisites..." + + # Check if running as non-root user + if [[ $EUID -eq 0 ]]; then + error "This script should not be run as root for security reasons" + exit 1 + fi + + # Check if inventory file exists + if [[ ! -f "$INVENTORY_FILE" ]]; then + error "Inventory file not found: $INVENTORY_FILE" + exit 1 + fi + + # Check if playbook exists + if [[ ! -f "$PLAYBOOK_FILE" ]]; then + error "Playbook file not found: $PLAYBOOK_FILE" + exit 1 + fi + + # Check if ansible is installed + if ! command -v ansible-playbook >/dev/null 2>&1; then + error "ansible-playbook is not installed" + exit 1 + fi + + # Check Python dependencies + if ! python3 -c "import bcrypt" >/dev/null 2>&1; then + warning "python3-bcrypt not found, installing..." + sudo apt-get update && sudo apt-get install -y python3-bcrypt + fi + + success "Prerequisites check passed" +} + +# Show configuration +show_configuration() { + log "Reading configuration from $INVENTORY_FILE..." + + # Extract configuration from inventory file + POSTGRES_VERSION=$(grep -E "^\s*#.*PostgreSQL Version:" "$INVENTORY_FILE" | awk '{print $NF}' || echo "15") + POSTGRES_PORT=$(grep -E "^\s*#.*PostgreSQL Port:" "$INVENTORY_FILE" | awk '{print $NF}' || echo "5433") + STORAGE_DEVICE=$(grep -E "^\s*#.*Storage Device:" "$INVENTORY_FILE" | awk '{print $NF}' || echo "/dev/nvme2n1") + MOUNT_POINT=$(grep -E "^\s*#.*Mount Point:" "$INVENTORY_FILE" | awk '{print $NF}' || echo "/srv/postgres") + TARGET_HOST=$(grep "ansible_host=" "$INVENTORY_FILE" | awk -F'ansible_host=' '{print $2}' | awk '{print $1}') + + cat << EOF + +Configuration Summary: + PostgreSQL Version: $POSTGRES_VERSION + Target Host: $TARGET_HOST + Port: $POSTGRES_PORT + Storage Device: $STORAGE_DEVICE + Mount Point: $MOUNT_POINT + Secrets Directory: $SECRETS_DIR + Inventory File: $INVENTORY_FILE + Playbook: $PLAYBOOK_FILE + +Security Features: + ✓ 16-character secure password generation + ✓ MD5 password encryption + ✓ Private subnet deployment (no SSL overhead) + ✓ Connection and statement audit logging + ✓ Kubernetes secrets with base64 encoding + ✓ Proper file permissions (0600 for secrets) + ✓ Separation of sensitive and non-sensitive data + +EOF +} + +# Test connectivity +test_connectivity() { + log "Testing connectivity to PostgreSQL servers..." + + if ansible postgresql_servers -i "$INVENTORY_FILE" -m ping >/dev/null 2>&1; then + success "Connectivity test passed" + else + error "Connectivity test failed" + echo "Please check your inventory file and SSH configuration" + exit 1 + fi +} + +# Clean up previous secrets +cleanup_previous_secrets() { + if [[ -d "$SECRETS_DIR" ]]; then + warning "Previous secrets directory found, cleaning up..." + rm -rf "$SECRETS_DIR" + fi + + # Create secure directory + mkdir -p "$SECRETS_DIR" + chmod 700 "$SECRETS_DIR" + log "Created secure secrets directory: $SECRETS_DIR" +} + +# Main execution +main() { + print_banner + + # Parse command line arguments + KUBERNETES_NAMESPACE="postgres" + SECRET_NAME="postgres-postgresql" + AUTO_CONFIRM=false + + while [[ $# -gt 0 ]]; do + case $1 in + --namespace) + KUBERNETES_NAMESPACE="$2" + shift 2 + ;; + --secret-name) + SECRET_NAME="$2" + shift 2 + ;; + --auto-confirm) + AUTO_CONFIRM=true + shift + ;; + -h|--help) + cat << EOF +PostgreSQL Secure Setup Script + +Usage: $0 [OPTIONS] + +Options: + --namespace NAMESPACE Kubernetes namespace for secrets (default: postgres) + --secret-name NAME Name for Kubernetes secret (default: postgres-postgresql) + --auto-confirm Skip confirmation prompts + -h, --help Show this help message + +Examples: + $0 + $0 --namespace production --secret-name postgres-creds + $0 --auto-confirm + +EOF + exit 0 + ;; + *) + error "Unknown option: $1" + exit 1 + ;; + esac + done + + check_prerequisites + show_configuration + test_connectivity + cleanup_previous_secrets + + # Confirmation prompt + if [[ "$AUTO_CONFIRM" == "false" ]]; then + echo + read -p "Do you want to proceed with PostgreSQL secure setup? (y/N): " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + log "Setup cancelled by user" + exit 0 + fi + fi + + log "Starting PostgreSQL secure setup..." + echo "==============================================" + + # Execute the secure playbook + if ansible-playbook -i "$INVENTORY_FILE" "$PLAYBOOK_FILE" \ + --extra-vars "kubernetes_namespace=$KUBERNETES_NAMESPACE" \ + --extra-vars "secret_name=$SECRET_NAME" \ + -v; then + + echo + echo "==============================================" + success "PostgreSQL secure setup completed successfully!" + + # Show generated files + if [[ -d "$SECRETS_DIR" ]]; then + echo + log "Generated Kubernetes files:" + ls -la "$SECRETS_DIR" + echo + success "Kubernetes secret files are ready for deployment" + success "Review the DEPLOYMENT_INSTRUCTIONS.md file for next steps" + + # Show file permissions + echo + log "File permissions (security check):" + find "$SECRETS_DIR" -type f -exec ls -l {} \; + + else + warning "Secrets directory not found. Check playbook execution." + fi + + else + error "PostgreSQL setup failed" + exit 1 + fi +} + +# Cleanup function for signals +cleanup() { + log "Script interrupted, cleaning up..." + # Add any cleanup operations here if needed + exit 1 +} + +# Set up signal handlers +trap cleanup SIGINT SIGTERM + +# Run main function +main "$@" diff --git a/Helmsman/utils/ansible/setup-vm-inventory.sh b/Helmsman/utils/ansible/setup-vm-inventory.sh new file mode 100755 index 00000000..b895ebc3 --- /dev/null +++ b/Helmsman/utils/ansible/setup-vm-inventory.sh @@ -0,0 +1,296 @@ +#!/bin/bash + +# VM Inventory Setup Script +# This script helps you create or update the Ansible inventory file for PostgreSQL setup + +set -e + +# Default values +INVENTORY_FILE="hosts.ini" +DEFAULT_USER="ubuntu" +DEFAULT_PORT="22" +DEFAULT_PG_VERSION="15" +DEFAULT_DEVICE="/dev/nvme2n1" +DEFAULT_MOUNT_POINT="/srv/postgres" +DEFAULT_NETWORK="10.1.0.0/16" +DEFAULT_PG_PORT="5432" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +print_usage() { + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " -f, --file FILE Inventory file name (default: hosts.ini)" + echo " -h, --help Show this help message" + echo "" + echo "This script will guide you through setting up your VM inventory." + echo "" + echo "Examples:" + echo " $0 # Interactive setup with default inventory file" + echo " $0 -f my-hosts.ini # Use custom inventory file name" +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -f|--file) + INVENTORY_FILE="$2" + shift 2 + ;; + -h|--help) + print_usage + exit 0 + ;; + *) + echo "Unknown option: $1" + print_usage + exit 1 + ;; + esac +done + +echo -e "${GREEN}=== VM Inventory Setup ===${NC}" +echo "This script will help you create the inventory file for PostgreSQL automation." +echo "" + +# Check if inventory file already exists +if [[ -f "$INVENTORY_FILE" ]]; then + echo -e "${YELLOW}Warning: Inventory file '$INVENTORY_FILE' already exists.${NC}" + echo "Current content:" + echo "----------------------------------------" + cat "$INVENTORY_FILE" + echo "----------------------------------------" + echo "" + read -p "Do you want to overwrite it? (y/N): " overwrite + if [[ $overwrite != [yY] && $overwrite != [yY][eE][sS] ]]; then + echo "Setup cancelled." + exit 0 + fi +fi + +# Collect VM information +echo -e "${BLUE}=== VM Connection Details ===${NC}" +echo "" + +# VM IP Address +while true; do + read -p "Enter your VM IP address: " vm_ip + if [[ -n "$vm_ip" ]]; then + break + else + echo -e "${RED}IP address cannot be empty.${NC}" + fi +done + +# SSH User +read -p "Enter SSH username (default: $DEFAULT_USER): " ssh_user +ssh_user="${ssh_user:-$DEFAULT_USER}" + +# SSH Key Path +echo "" +echo "SSH Key Configuration:" +echo "Please provide the path to your SSH private key file." +echo "Common locations:" +echo " • ~/.ssh/id_rsa" +echo " • ~/.ssh/your-key.pem" +echo " • ~/Downloads/your-aws-key.pem" +echo "" + +while true; do + read -p "Enter SSH private key file path: " ssh_key + if [[ -n "$ssh_key" ]]; then + # Expand tilde to home directory + ssh_key="${ssh_key/#\~/$HOME}" + + if [[ -f "$ssh_key" ]]; then + break + else + echo -e "${RED}File not found: $ssh_key${NC}" + echo "Please check the path and try again." + fi + else + echo -e "${RED}SSH key path cannot be empty.${NC}" + fi +done + +# SSH Port (optional) +read -p "Enter SSH port (default: $DEFAULT_PORT): " ssh_port +ssh_port="${ssh_port:-$DEFAULT_PORT}" + +# Server Name +read -p "Enter server name for inventory (default: vm_server): " server_name +server_name="${server_name:-vm_server}" + +echo "" +echo -e "${BLUE}=== PostgreSQL Configuration ===${NC}" +echo "These parameters will be used for PostgreSQL installation:" +echo "" + +# PostgreSQL Version +read -p "Enter PostgreSQL version (default: $DEFAULT_PG_VERSION): " pg_version +pg_version="${pg_version:-$DEFAULT_PG_VERSION}" + +# Storage Device +echo "" +echo "Storage Device Configuration:" +echo "This is the block device where PostgreSQL data will be stored." +echo "Common devices: /dev/sdb, /dev/nvme2n1, /dev/xvdf" +echo "" +read -p "Enter storage device path (default: $DEFAULT_DEVICE): " storage_device +storage_device="${storage_device:-$DEFAULT_DEVICE}" + +# Mount Point +echo "" +echo "Mount Point Configuration:" +echo "This is where the storage device will be mounted for PostgreSQL data." +echo "" +read -p "Enter mount point (default: $DEFAULT_MOUNT_POINT): " mount_point +mount_point="${mount_point:-$DEFAULT_MOUNT_POINT}" + +# Network CIDR +echo "" +echo "Network Configuration:" +echo "This is the network CIDR for PostgreSQL access control." +echo "Examples: 10.1.0.0/16, 192.168.1.0/24, 172.16.0.0/12" +echo "" +read -p "Enter network CIDR (default: $DEFAULT_NETWORK): " network_cidr +network_cidr="${network_cidr:-$DEFAULT_NETWORK}" + + +# PostgreSQL Port +echo "" +echo "PostgreSQL Port Configuration:" +echo "This is the port PostgreSQL will listen on." +echo "Default is 5432, but you might want to use a different port for security." +echo "" +read -p "Enter PostgreSQL port (default: $DEFAULT_PG_PORT): " pg_port +pg_port="${pg_port:-$DEFAULT_PG_PORT}" +echo "" +echo -e "${BLUE}=== Configuration Summary ===${NC}" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "VM Connection:" +echo " Server Name: $server_name" +echo " IP Address: $vm_ip" +echo " SSH User: $ssh_user" +echo " SSH Key: $ssh_key" +echo " SSH Port: $ssh_port" +echo "" +echo "PostgreSQL Configuration:" +echo " Version: $pg_version" +echo " Storage Device: $storage_device" +echo " Mount Point: $mount_point" +echo " Network CIDR: $network_cidr" +echo " PostgreSQL Port: $pg_port" +echo "" +echo "Inventory File: $INVENTORY_FILE" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo "" + +read -p "Create inventory file with these settings? (y/N): " confirm +if [[ $confirm != [yY] && $confirm != [yY][eE][sS] ]]; then + echo "Setup cancelled." + exit 0 +fi + +# Create the inventory file +echo -e "${GREEN}Creating inventory file...${NC}" + +cat > "$INVENTORY_FILE" << INVENTORY_EOF +# Ansible Inventory for PostgreSQL Setup on Virtual Machine +# Generated by setup-vm-inventory.sh on $(date) +# +# Configuration Summary: +# - PostgreSQL Version: $pg_version +# - Storage Device: $storage_device +# - Mount Point: $mount_point +# - Network CIDR: $network_cidr +# - PostgreSQL Port: $pg_port + +[postgresql_servers] +$server_name ansible_host=$vm_ip ansible_user=$ssh_user ansible_ssh_private_key_file=$ssh_key ansible_port=$ssh_port + +[postgresql_servers:vars] +# SSH and Ansible Configuration +ansible_become=yes +ansible_become_method=sudo +ansible_ssh_common_args='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' +ansible_python_interpreter=/usr/bin/python3 +ansible_ssh_timeout=30 +ansible_connect_timeout=30 + +# PostgreSQL Configuration Variables +postgresql_version=$pg_version +storage_device=$storage_device +mount_point=$mount_point +network_cidr=$network_cidr + +# Additional PostgreSQL Variables +postgresql_port=$pg_port + +# For local testing (if needed) +[local] +localhost ansible_connection=local +INVENTORY_EOF + +echo -e "${GREEN}✓ Inventory file '$INVENTORY_FILE' created successfully!${NC}" +echo "" + +# Test connectivity +echo -e "${BLUE}=== Testing Connectivity ===${NC}" +echo "Testing SSH connection to your VM..." + +if command -v ansible &> /dev/null; then + if ansible postgresql_servers -i "$INVENTORY_FILE" -m ping; then + echo -e "${GREEN}✓ Connection test successful!${NC}" + echo "" + echo -e "${GREEN}=== Setup Complete ===${NC}" + echo "Your inventory is ready with the following configuration:" + echo " • PostgreSQL Version: $pg_version" + echo " • Storage Device: $storage_device" + echo " • Mount Point: $mount_point" + echo " • Network CIDR: $network_cidr" +echo " • PostgreSQL Port: $pg_port" + echo "" + echo "You can now run your PostgreSQL automation scripts:" + echo " ./ansible-postgresql.sh" + echo "" + echo "Or test with manual commands like:" + echo " ansible-playbook -i $INVENTORY_FILE your-playbook.yml" + echo "" + else + echo -e "${RED}✗ Connection test failed${NC}" + echo "" + echo -e "${YELLOW}=== Troubleshooting Tips ===${NC}" + echo "1. Check if your VM is running and accessible" + echo "2. Verify the IP address is correct" + echo "3. Ensure SSH key has correct permissions (600)" + echo "4. Check if the SSH user has sudo privileges" + echo "5. Verify firewall/security groups allow SSH access" + echo "6. Make sure the storage device exists on the target VM" + echo "" + echo "You can test manually with:" + echo " ssh -i $ssh_key $ssh_user@$vm_ip" + echo "" + echo "The inventory file has been created, but please fix connectivity issues before proceeding." + fi +else + echo -e "${YELLOW}⚠ Ansible not found - skipping connectivity test${NC}" + echo "Install Ansible to test connectivity: sudo apt install ansible" + echo "" + echo -e "${GREEN}=== Setup Complete ===${NC}" + echo "Inventory file created with your configuration." +fi + +echo "" +echo -e "${BLUE}=== Inventory File Created ===${NC}" +echo "File: $INVENTORY_FILE" +echo "Location: $(pwd)/$INVENTORY_FILE" +echo "" +echo "You can review the inventory file with:" +echo " cat $INVENTORY_FILE" diff --git a/Helmsman/utils/config-server-values.yaml b/Helmsman/utils/config-server-values.yaml index c4a1d78e..c2787ba3 100644 --- a/Helmsman/utils/config-server-values.yaml +++ b/Helmsman/utils/config-server-values.yaml @@ -1,6 +1,6 @@ gitRepo: uri: https://github.com/mosip/mosip-config - version: release-1.2.4.3 + version: 1.2.4.3-beta ## Folders within the base repo where properties may be found. searchFolders: "" private: false diff --git a/terraform/implementations/aws/infra/aws.tfvars b/terraform/implementations/aws/infra/aws.tfvars index 59a75677..5e86c83f 100644 --- a/terraform/implementations/aws/infra/aws.tfvars +++ b/terraform/implementations/aws/infra/aws.tfvars @@ -29,7 +29,7 @@ ami = "ami-xxxxxxxxxxxx" # Ubuntu 24.04 LTS AMI ID for ap-south-1 # Repo K8S-INFRA URL k8s_infra_repo_url = "https://github.com/mosip/k8s-infra.git" # Repo K8S-INFRA branch -k8s_infra_branch = "" +k8s_infra_branch = "v1.2.1.0" # NGINX Node's Root volume size nginx_node_root_volume_size = 24 # NGINX node's EBS volume size @@ -70,11 +70,12 @@ mount_point = "/srv/postgres" postgresql_port = "5433" # MOSIP Infrastructure Repository Configuration -mosip_infra_repo_url = "https://github.com/mosip/mosip-infra.git" +mosip_infra_repo_url = "https://github.com/mosip/infra.git" -mosip_infra_branch = "" +mosip_infra_branch = "v0.1.0-beta.1" # VPC Configuration - Existing VPC to use (discovered by Name tag) vpc_name = "" +v \ No newline at end of file diff --git a/terraform/modules/aws/postgresql-setup/postgresql-setup.sh b/terraform/modules/aws/postgresql-setup/postgresql-setup.sh index 836d89fe..9d8471b8 100755 --- a/terraform/modules/aws/postgresql-setup/postgresql-setup.sh +++ b/terraform/modules/aws/postgresql-setup/postgresql-setup.sh @@ -254,7 +254,7 @@ ansible --version || echo "Ansible: Not available" # Clone MOSIP infrastructure repository with retry logic echo 'Cloning Repository...' cd /tmp -rm -rf mosip-infra +rm -rf infra echo "Cloning from: $MOSIP_INFRA_REPO_URL" echo "Branch: $MOSIP_INFRA_BRANCH" @@ -270,7 +270,7 @@ git clone "$MOSIP_INFRA_REPO_URL" || { } } -cd mosip-infra +cd infra git checkout "$MOSIP_INFRA_BRANCH" || { echo "Branch checkout failed for branch: $MOSIP_INFRA_BRANCH" echo 'Available branches:' @@ -283,13 +283,13 @@ echo "Successfully cloned and checked out branch: $MOSIP_INFRA_BRANCH" # Navigate to PostgreSQL Ansible directory echo 'Navigating to PostgreSQL Ansible Directory...' echo 'Current directory structure:' -find /tmp/mosip-infra -name '*postgres*' -type d 2>/dev/null || echo 'No postgres directories found' +find /tmp/infra -name '*ansible*' -type d 2>/dev/null || echo 'No postgres directories found' -POSTGRES_ANSIBLE_DIR="/tmp/mosip-infra/deployment/v3/external/postgres/ansible" +POSTGRES_ANSIBLE_DIR="/tmp/infra/Helmsman/utils/ansible" if [ ! -d "$POSTGRES_ANSIBLE_DIR" ]; then echo "PostgreSQL Ansible directory not found at: $POSTGRES_ANSIBLE_DIR" echo 'Available directories under deployment:' - find /tmp/mosip-infra -name 'deployment' -type d -exec find {} -type d \; 2>/dev/null | head -20 + find /tmp/infra -name 'utils' -type d -exec find {} -type d \; 2>/dev/null | head -20 exit 1 fi @@ -884,7 +884,7 @@ echo '' echo '=== [CLEANUP] Cleaning up temporary files ===' # Clean up sensitive information echo '[CLEAN] Cleaning up temporary files...' -rm -rf /tmp/mosip-infra 2>/dev/null || true +rm -rf /tmp/infra 2>/dev/null || true rm -f /tmp/postgresql-ansible.log 2>/dev/null || true # Keep the generated YAML files for reference (they will be cleaned up by control plane script)