diff --git a/README.md b/README.md index 99f1d52..5689586 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,9 @@ Some notes: - Use option `--java-opts="JAVA_OPTS"` to override the default JAVA_OPTS for the Tomcat process. That's tipically used to set the maximum/initial Heap Memory size (for example: `--java-opts="-Xmx3500m -Xms2500m"`) - Use option `--postgis-version=13-3.1-alpine` to specify the PostGIS version to use. By default, 10-2.5-alpine is used. - Use option `--debug-port=PORT` to specify the debug port of the Tomcat process. +- Use option `--external-db-volume=VOLUME_ABSOLUTE_PATH` to create or use a persistant volume for the database located at `VOLUME_ABSOLUTE_PATH`. +- Use option `--external-db-url=POSTGRES_URL` to connect to an external PostgreSQL database instead of using the DB container. The URL should be in the format `postgresql://USER:PASSWORD@HOST:PORT/DBNAME`. Note that you have configure the DB to accept connections from the docker internal network. +- Use option `--load-dump-from-data` with `--external-db-url` to import the SQL dump to the external database. Equivalent to running without `-k`/`--keep-containers`. The recieving DB should have the appropiate config (DB owner, user permissions, postgis extention). #### Custom DHIS2 dhis.conf diff --git a/src/d2_docker/commands/start.py b/src/d2_docker/commands/start.py index bfa72b3..dd64522 100644 --- a/src/d2_docker/commands/start.py +++ b/src/d2_docker/commands/start.py @@ -41,8 +41,23 @@ def setup(parser): parser.add_argument("--postgis-version", type=str, help="Set PostGIS database version") parser.add_argument("--enable-postgres-queries-logging", action="store_true", help="Enable Postgres queries logging") - parser.add_argument("--glowroot-port", metavar="PORT", help="Set glowroot port") + parser.add_argument( + "--external-db-volume", + metavar="DIRECTORY", + help="Directory for external database volume", + ) + parser.add_argument( + "--external-db-url", + type=str, + metavar="postgresql://user:pass@host:port/dbname", + help="Use external PostgreSQL database" + ) + parser.add_argument( + "--load-dump-from-data", + action="store_true", + help="Load database dump from data container (only with --external-db-url)", + ) def run(args): @@ -54,9 +69,25 @@ def run(args): image2 = args.image args.image = image2 + + if args.external_db_volume: + check_db_volume_path(args.external_db_volume) + + if args.external_db_url: + utils.validate_external_db_connection(args.external_db_url) + start(args) +def check_db_volume_path(external_db_volume): + if not os.path.isabs(external_db_volume): + msg = "--external-db-volume must be an absolute path: {}".format(external_db_volume) + raise utils.D2DockerError(msg) + if not os.path.exists(external_db_volume): + msg = "--external-db-volume path does not exist: {}".format(external_db_volume) + raise utils.D2DockerError(msg) + + def import_from_file(images_path): dhis2_data_image_re = "/{}:".format(utils.DHIS2_DATA_IMAGE) result = utils.load_images_file(images_path) @@ -85,10 +116,13 @@ def start(args): override_containers = not args.keep_containers if args.pull: - utils.run_docker_compose(["pull"], image_name, core_image=core_image) + utils.run_docker_compose(["pull"], image_name, core_image=core_image, + external_db_volume=args.external_db_volume) if override_containers: - utils.run_docker_compose(["down", "--volumes"], image_name, core_image=core_image) + utils.run_docker_compose(["down", "--volumes"], image_name, core_image=core_image, + external_db_volume=args.external_db_volume, + external_db_url=args.external_db_url) up_args = filter( bool, ["--force-recreate" if override_containers else None, "-d" if args.detach else None] @@ -115,7 +149,10 @@ def start(args): java_opts=args.java_opts, postgis_version=args.postgis_version, enable_postgres_queries_logging=args.enable_postgres_queries_logging, - glowroot_port=args.glowroot_port + glowroot_port=args.glowroot_port, + external_db_volume=args.external_db_volume, + external_db_url=args.external_db_url, + load_dump_from_data=args.load_dump_from_data, ) if args.detach: diff --git a/src/d2_docker/config/dhis2-core-start.sh b/src/d2_docker/config/dhis2-core-start.sh index 508fcde..7d87810 100755 --- a/src/d2_docker/config/dhis2-core-start.sh +++ b/src/d2_docker/config/dhis2-core-start.sh @@ -10,17 +10,23 @@ set -e -u -o pipefail # # Global: LOAD_FROM_DATA="yes" | "no" +# Global: LOAD_DUMP_FROM_DATA="yes" | "no" +# Global: EXTERNAL_DB_URL=string (optional) # Global: DEPLOY_PATH=string # Global: DHIS2_AUTH=string export PGPASSWORD="dhis" +db_url="" +if [[ -n "$EXTERNAL_DB_URL" ]]; then + db_url="${EXTERNAL_DB_URL//localhost/host.docker.internal}" +fi dhis2_url="http://localhost:8080/$DEPLOY_PATH" dhis2_url_with_auth="http://$DHIS2_AUTH@localhost:8080/$DEPLOY_PATH" -psql_base_cmd="psql --quiet -h db -U dhis dhis2" +psql_base_cmd="psql --quiet ${db_url:-"-h db -U dhis dhis2"}" psql_cmd="$psql_base_cmd -v ON_ERROR_STOP=0" psql_strict_cmd="$psql_base_cmd -v ON_ERROR_STOP=1" -pgrestore_cmd="pg_restore -h db -U dhis -d dhis2" +pgrestore_cmd="pg_restore ${db_url:-"-h db -U dhis dhis2"}" configdir="/config" homedir="/dhis2-home-files" scripts_dir="/data/scripts" @@ -37,7 +43,12 @@ debug() { } run_sql_files() { - base_db_path=$(test "${LOAD_FROM_DATA}" = "yes" && echo "$root_db_path" || echo "$post_db_path") + if { [ -z "$db_url" ] && [ "${LOAD_FROM_DATA}" = "yes" ]; } || + { [ -n "$db_url" ] && [ "${LOAD_FROM_DATA}" = "yes" ] && [ "${LOAD_DUMP_FROM_DATA}" = "yes" ]; }; then + base_db_path="$root_db_path" + else + base_db_path="$post_db_path" + fi debug "Files in data path" find "$base_db_path" >&2 @@ -69,10 +80,10 @@ run_psql_cmd() { local path=$1 if [[ "$path" == *strict* ]]; then echo "Strict mode: $path" - $psql_strict_cmd < "$path" + $psql_strict_cmd <"$path" else echo "Normal mode: $path" - $psql_cmd < "$path" + $psql_cmd <"$path" fi } diff --git a/src/d2_docker/docker-compose.yml b/src/d2_docker/docker-compose.yml index f0f8d22..80b2ec6 100644 --- a/src/d2_docker/docker-compose.yml +++ b/src/d2_docker/docker-compose.yml @@ -17,6 +17,8 @@ services: CATALINA_OPTS: "-Dcontext.path=${DEPLOY_PATH} -Xdebug -Xrunjdwp:transport=dt_socket,address=0.0.0.0:8000,server=y,suspend=n" JAVA_OPTS: "-Xmx7500m -Xms4000m ${JAVA_OPTS}" LOAD_FROM_DATA: "${LOAD_FROM_DATA}" + EXTERNAL_DB_URL: "${EXTERNAL_DB_URL}" + LOAD_DUMP_FROM_DATA: "${LOAD_DUMP_FROM_DATA}" DEPLOY_PATH: "${DEPLOY_PATH}" DHIS2_AUTH: "${DHIS2_AUTH}" entrypoint: bash /config/dhis2-core-entrypoint.sh diff --git a/src/d2_docker/utils.py b/src/d2_docker/utils.py index 138c372..127eca1 100644 --- a/src/d2_docker/utils.py +++ b/src/d2_docker/utils.py @@ -10,9 +10,10 @@ import time import yaml import urllib.request +from urllib.parse import urlparse from setuptools._distutils import dir_util from pathlib import Path -from typing import Optional +from typing import Dict, Optional import d2_docker from d2_docker.glowroot import get_port_glowroot @@ -25,6 +26,7 @@ PROJECT_DIR = os.path.dirname(os.path.realpath(__file__)) ROOT_PATH = os.environ.get("ROOT_PATH") or PROJECT_DIR + def get_dhis2_war_url(version): match = (re.match(r"^(\d+.\d+)", version) if version.startswith("2.") else re.match(r"^(\d+)", version)) @@ -263,6 +265,9 @@ def run_docker_compose( postgis_version=None, enable_postgres_queries_logging=False, glowroot_port=None, + external_db_volume=None, + external_db_url=None, + load_dump_from_data=False, **kwargs, ): """ @@ -279,6 +284,19 @@ def run_docker_compose( post_sql_dir_abs = get_absdir_for_docker_volume(post_sql_dir) scripts_dir_abs = get_absdir_for_docker_volume(scripts_dir) + if external_db_url and not dhis_conf: + logger.info("External DB URL provided, updating dhis.conf") + db_config = parse_postgres_url(external_db_url) + if not db_config: + raise D2DockerError("Invalid PostgreSQL URL format") + + dhis_conf_file = build_dhis_conf_from_external_db( + jdbc_url=db_config['url'], + username=db_config['user'], + password=db_config['password'] + ) + dhis_conf = dhis_conf_file.name + env_pairs = [ ("DHIS2_DATA_IMAGE", final_image_name), ("DHIS2_CORE_PORT", str(port)) if port else None, @@ -300,6 +318,9 @@ def run_docker_compose( ("ROOT_PATH", ROOT_PATH), ("PSQL_ENABLE_QUERY_LOGS", "") if not enable_postgres_queries_logging else None, ("GLOWROOT_PORT", get_port_glowroot(glowroot_port)) + ("EXTERNAL_DB_VOLUME", external_db_volume) if external_db_volume else None, + ("EXTERNAL_DB_URL", external_db_url) if external_db_url else None, + ("LOAD_DUMP_FROM_DATA", "yes" if load_dump_from_data else "no") if external_db_url else "no", ] env = dict((k, v) for (k, v) in [pair for pair in env_pairs if pair] if v is not None) @@ -310,6 +331,21 @@ def process_yaml(data): for env_port in env_ports: if env_port not in env: core["ports"] = [port for port in core["ports"] if env_port not in port] + if "EXTERNAL_DB_VOLUME" in env: + data["volumes"]["pgdata"] = { + 'driver': 'local', + 'driver_opts': { + 'type': 'none', + 'o': 'bind', + 'device': external_db_volume + } + } + if "EXTERNAL_DB_URL" in env: + del data["services"]["db"] + del data["volumes"]["pgdata"] + core = data["services"]["core"] + core["depends_on"] = [dep for dep in core["depends_on"] if dep != "db"] + core["extra_hosts"] = ["host.docker.internal:host-gateway"] return data @@ -335,6 +371,95 @@ def build_docker_compose(process_yaml): return temp_compose + +def validate_external_db_connection(db_url): + """Validate connection to external PostgreSQL database.""" + logger.info("Validating external database connection...") + try: + psql_cmd = ["psql", "-d", db_url, "-c", "SELECT now();", "&> /dev/null"] + run(psql_cmd, capture_output=False) + logger.info("External database validation successful") + except Exception as e: + raise D2DockerError(f"External database validation failed: {e}") + + +def parse_postgres_url(url: str) -> Optional[Dict[str, str]]: + """Parse PostgreSQL connection URL. + + Args: + url (str): PostgreSQL connection URL in the format postgresql://user:pass@host:port/dbname + + Raises: + D2DockerError: If the URL is invalid. + + Returns: + Optional[Dict[str, str]]: Dictionary with keys 'url', 'user', 'password' or None if url is empty. + """ + if not url: + return None + + try: + parsed = urlparse(url) + if parsed.scheme not in ['postgresql']: + raise D2DockerError(f"Invalid PostgreSQL URL scheme: {parsed.scheme}") + + if None in [parsed.username, parsed.password, parsed.hostname, parsed.path]: + raise D2DockerError(f"Missing components in PostgreSQL URL: {url}") + + if parsed.hostname == "localhost": + hostname = "host.docker.internal" + else: + hostname = parsed.hostname + + port = ":"+str(parsed.port) if parsed.port else "" + + return { + 'url': "jdbc:"+parsed.scheme+"://"+str(hostname)+port+parsed.path, + 'user': str(parsed.username), + 'password': str(parsed.password), + } + except Exception as e: + raise D2DockerError(f"Failed to parse PostgreSQL URL: {e}") + + +def build_dhis_conf_from_external_db(jdbc_url, username, password): + base_config_path = os.path.join(ROOT_PATH, "config", "DHIS2_home", "dhis.conf") + with open(base_config_path, 'r') as file: + config = file.read() + + # Update connection URL + config = re.sub( + r'^connection\.url\s*=.*$', + f'connection.url = {jdbc_url}', + config, + flags=re.MULTILINE + ) + + if username: + config = re.sub( + r'^connection\.username\s*=.*$', + f'connection.username = {username}', + config, + flags=re.MULTILINE + ) + + if password: + config = re.sub( + r'^connection\.password\s*=.*$', + f'connection.password = {password}', + config, + flags=re.MULTILINE + ) + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.conf') as temp_conf: + temp_conf.write(config) + + os.chmod(temp_conf.name, 0o644) + + atexit.register(lambda: os.remove(temp_conf.name)) + return temp_conf + + def get_config_path(default_filename, path): return os.path.abspath(path) if path else get_config_file(default_filename) @@ -470,6 +595,7 @@ def export_data_from_image(source_image, dest_path): # https://github.com/dhis2/dhis2-core/blob/master/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceDomain.java#L35 + default_folders = [ "apps", "dataValue", @@ -482,6 +608,7 @@ def export_data_from_image(source_image, dest_path): "jobData" ] + def export_data_from_running_containers(image_name, containers, destination, folders=None): """Export data (db + apps + documents) from a running Docker container to some folder.""" logger.info("Copy Dhis2 apps")