From 7573e692ff5483efbe3c2fc2a2ae07d09ac190be Mon Sep 17 00:00:00 2001 From: nshandra <34254522+nshandra@users.noreply.github.com> Date: Fri, 7 Nov 2025 09:54:50 +0100 Subject: [PATCH 1/2] feat: add external persisted volume for DB container --- src/d2_docker/commands/start.py | 25 +++++++++++++++++++++++-- src/d2_docker/utils.py | 11 +++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/d2_docker/commands/start.py b/src/d2_docker/commands/start.py index 2ff59f4..cb03af5 100644 --- a/src/d2_docker/commands/start.py +++ b/src/d2_docker/commands/start.py @@ -41,6 +41,11 @@ def setup(parser): parser.add_argument("--postgis-version", type=str, help="Set PostGIS database version") parser.add_argument("--enable-postgres-queries-logging", action="store_true", help="Enable Postgres queries logging") + parser.add_argument( + "--external-db-volume", + metavar="DIRECTORY", + help="Directory for external database volume", + ) def run(args): @@ -52,9 +57,22 @@ def run(args): image2 = args.image args.image = image2 + + if args.external_db_volume: + check_db_volume_path(args.external_db_volume) + start(args) +def check_db_volume_path(external_db_volume): + if not os.path.isabs(external_db_volume): + msg = "--external-db-volume must be an absolute path: {}".format(external_db_volume) + raise utils.D2DockerError(msg) + if not os.path.exists(external_db_volume): + msg = "--external-db-volume path does not exist: {}".format(external_db_volume) + raise utils.D2DockerError(msg) + + def import_from_file(images_path): dhis2_data_image_re = "/{}:".format(utils.DHIS2_DATA_IMAGE) result = utils.load_images_file(images_path) @@ -83,10 +101,12 @@ def start(args): override_containers = not args.keep_containers if args.pull: - utils.run_docker_compose(["pull"], image_name, core_image=core_image) + utils.run_docker_compose(["pull"], image_name, core_image=core_image, + external_db_volume=args.external_db_volume) if override_containers: - utils.run_docker_compose(["down", "--volumes"], image_name, core_image=core_image) + utils.run_docker_compose(["down", "--volumes"], image_name, core_image=core_image, + external_db_volume=args.external_db_volume) up_args = filter( bool, ["--force-recreate" if override_containers else None, "-d" if args.detach else None] @@ -113,6 +133,7 @@ def start(args): java_opts=args.java_opts, postgis_version=args.postgis_version, enable_postgres_queries_logging=args.enable_postgres_queries_logging, + external_db_volume=args.external_db_volume, ) if args.detach: diff --git a/src/d2_docker/utils.py b/src/d2_docker/utils.py index c92cec4..1d5ec85 100644 --- a/src/d2_docker/utils.py +++ b/src/d2_docker/utils.py @@ -261,6 +261,7 @@ def run_docker_compose( tomcat_server=None, postgis_version=None, enable_postgres_queries_logging=False, + external_db_volume=None, **kwargs, ): """ @@ -296,6 +297,7 @@ def run_docker_compose( # Add ROOT_PATH from environment (required when run inside a docker) ("ROOT_PATH", ROOT_PATH), ("PSQL_ENABLE_QUERY_LOGS", "") if not enable_postgres_queries_logging else None, + ("EXTERNAL_DB_VOLUME", external_db_volume) if external_db_volume else None, ] env = dict((k, v) for (k, v) in [pair for pair in env_pairs if pair] if v is not None) @@ -303,6 +305,15 @@ def process_yaml(data): if "DHIS2_CORE_DEBUG_PORT" not in env: core = data["services"]["core"] core["ports"] = [port for port in core["ports"] if "DHIS2_CORE_DEBUG_PORT" not in port] + if "EXTERNAL_DB_VOLUME" in env: + data["volumes"]["pgdata"] = { + 'driver': 'local', + 'driver_opts': { + 'type': 'none', + 'o': 'bind', + 'device': external_db_volume + } + } return data From 2cb916a4422a7d9fa2b835080a53f74f3c2fca52 Mon Sep 17 00:00:00 2001 From: nshandra <34254522+nshandra@users.noreply.github.com> Date: Fri, 28 Nov 2025 12:15:29 +0100 Subject: [PATCH 2/2] feat: add external DB connection option to start --- README.md | 3 + src/d2_docker/commands/start.py | 19 +++- src/d2_docker/config/dhis2-core-start.sh | 21 +++- src/d2_docker/docker-compose.yml | 2 + src/d2_docker/utils.py | 122 ++++++++++++++++++++++- 5 files changed, 158 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 0b1c2b8..ea39c87 100644 --- a/README.md +++ b/README.md @@ -92,6 +92,9 @@ Some notes: - Use option `--java-opts="JAVA_OPTS"` to override the default JAVA_OPTS for the Tomcat process. That's tipically used to set the maximum/initial Heap Memory size (for example: `--java-opts="-Xmx3500m -Xms2500m"`) - Use option `--postgis-version=13-3.1-alpine` to specify the PostGIS version to use. By default, 10-2.5-alpine is used. - Use option `--debug-port=PORT` to specify the debug port of the Tomcat process. +- Use option `--external-db-volume=VOLUME_ABSOLUTE_PATH` to create or use a persistant volume for the database located at `VOLUME_ABSOLUTE_PATH`. +- Use option `--external-db-url=POSTGRES_URL` to connect to an external PostgreSQL database instead of using the DB container. The URL should be in the format `postgresql://USER:PASSWORD@HOST:PORT/DBNAME`. Note that you have configure the DB to accept connections from the docker internal network. +- Use option `--load-dump-from-data` with `--external-db-url` to import the SQL dump to the external database. Equivalent to running without `-k`/`--keep-containers`. The recieving DB should have the appropiate config (DB owner, user permissions, postgis extention). #### Custom DHIS2 dhis.conf diff --git a/src/d2_docker/commands/start.py b/src/d2_docker/commands/start.py index cb03af5..9187130 100644 --- a/src/d2_docker/commands/start.py +++ b/src/d2_docker/commands/start.py @@ -46,6 +46,17 @@ def setup(parser): metavar="DIRECTORY", help="Directory for external database volume", ) + parser.add_argument( + "--external-db-url", + type=str, + metavar="postgresql://user:pass@host:port/dbname", + help="Use external PostgreSQL database" + ) + parser.add_argument( + "--load-dump-from-data", + action="store_true", + help="Load database dump from data container (only with --external-db-url)", + ) def run(args): @@ -61,6 +72,9 @@ def run(args): if args.external_db_volume: check_db_volume_path(args.external_db_volume) + if args.external_db_url: + utils.validate_external_db_connection(args.external_db_url) + start(args) @@ -106,7 +120,8 @@ def start(args): if override_containers: utils.run_docker_compose(["down", "--volumes"], image_name, core_image=core_image, - external_db_volume=args.external_db_volume) + external_db_volume=args.external_db_volume, + external_db_url=args.external_db_url) up_args = filter( bool, ["--force-recreate" if override_containers else None, "-d" if args.detach else None] @@ -134,6 +149,8 @@ def start(args): postgis_version=args.postgis_version, enable_postgres_queries_logging=args.enable_postgres_queries_logging, external_db_volume=args.external_db_volume, + external_db_url=args.external_db_url, + load_dump_from_data=args.load_dump_from_data, ) if args.detach: diff --git a/src/d2_docker/config/dhis2-core-start.sh b/src/d2_docker/config/dhis2-core-start.sh index 4a55c5b..ced0286 100755 --- a/src/d2_docker/config/dhis2-core-start.sh +++ b/src/d2_docker/config/dhis2-core-start.sh @@ -10,17 +10,23 @@ set -e -u -o pipefail # # Global: LOAD_FROM_DATA="yes" | "no" +# Global: LOAD_DUMP_FROM_DATA="yes" | "no" +# Global: EXTERNAL_DB_URL=string (optional) # Global: DEPLOY_PATH=string # Global: DHIS2_AUTH=string export PGPASSWORD="dhis" +db_url="" +if [[ -n "$EXTERNAL_DB_URL" ]]; then + db_url="${EXTERNAL_DB_URL//localhost/host.docker.internal}" +fi dhis2_url="http://localhost:8080/$DEPLOY_PATH" dhis2_url_with_auth="http://$DHIS2_AUTH@localhost:8080/$DEPLOY_PATH" -psql_base_cmd="psql --quiet -h db -U dhis dhis2" +psql_base_cmd="psql --quiet ${db_url:-"-h db -U dhis dhis2"}" psql_cmd="$psql_base_cmd -v ON_ERROR_STOP=0" psql_strict_cmd="$psql_base_cmd -v ON_ERROR_STOP=1" -pgrestore_cmd="pg_restore -h db -U dhis -d dhis2" +pgrestore_cmd="pg_restore ${db_url:-"-h db -U dhis dhis2"}" configdir="/config" homedir="/dhis2-home-files" scripts_dir="/data/scripts" @@ -37,7 +43,12 @@ debug() { } run_sql_files() { - base_db_path=$(test "${LOAD_FROM_DATA}" = "yes" && echo "$root_db_path" || echo "$post_db_path") + if { [ -z "$db_url" ] && [ "${LOAD_FROM_DATA}" = "yes" ]; } || + { [ -n "$db_url" ] && [ "${LOAD_FROM_DATA}" = "yes" ] && [ "${LOAD_DUMP_FROM_DATA}" = "yes" ]; }; then + base_db_path="$root_db_path" + else + base_db_path="$post_db_path" + fi debug "Files in data path" find "$base_db_path" >&2 @@ -69,10 +80,10 @@ run_psql_cmd() { local path=$1 if [[ "$path" == *strict* ]]; then echo "Strict mode: $path" - $psql_strict_cmd < "$path" + $psql_strict_cmd <"$path" else echo "Normal mode: $path" - $psql_cmd < "$path" + $psql_cmd <"$path" fi } diff --git a/src/d2_docker/docker-compose.yml b/src/d2_docker/docker-compose.yml index bf6d447..ab874b2 100644 --- a/src/d2_docker/docker-compose.yml +++ b/src/d2_docker/docker-compose.yml @@ -17,6 +17,8 @@ services: CATALINA_OPTS: "-Dcontext.path=${DEPLOY_PATH} -Xdebug -Xrunjdwp:transport=dt_socket,address=0.0.0.0:8000,server=y,suspend=n" JAVA_OPTS: "-Xmx7500m -Xms4000m ${JAVA_OPTS}" LOAD_FROM_DATA: "${LOAD_FROM_DATA}" + EXTERNAL_DB_URL: "${EXTERNAL_DB_URL}" + LOAD_DUMP_FROM_DATA: "${LOAD_DUMP_FROM_DATA}" DEPLOY_PATH: "${DEPLOY_PATH}" DHIS2_AUTH: "${DHIS2_AUTH}" entrypoint: bash /config/dhis2-core-entrypoint.sh diff --git a/src/d2_docker/utils.py b/src/d2_docker/utils.py index 1d5ec85..b59fee2 100644 --- a/src/d2_docker/utils.py +++ b/src/d2_docker/utils.py @@ -8,11 +8,12 @@ import socket import tempfile import time -import yaml +from urllib.parse import urlparse import urllib.request -from setuptools._distutils import dir_util from pathlib import Path -from typing import Optional +from typing import Dict, Optional +import yaml +from setuptools._distutils import dir_util import d2_docker from .image_name import ImageName @@ -24,6 +25,7 @@ PROJECT_DIR = os.path.dirname(os.path.realpath(__file__)) ROOT_PATH = os.environ.get("ROOT_PATH") or PROJECT_DIR + def get_dhis2_war_url(version): match = (re.match(r"^(\d+.\d+)", version) if version.startswith("2.") else re.match(r"^(\d+)", version)) @@ -262,6 +264,8 @@ def run_docker_compose( postgis_version=None, enable_postgres_queries_logging=False, external_db_volume=None, + external_db_url=None, + load_dump_from_data=False, **kwargs, ): """ @@ -277,6 +281,19 @@ def run_docker_compose( post_sql_dir_abs = get_absdir_for_docker_volume(post_sql_dir) scripts_dir_abs = get_absdir_for_docker_volume(scripts_dir) + if external_db_url and not dhis_conf: + logger.info("External DB URL provided, updating dhis.conf") + db_config = parse_postgres_url(external_db_url) + if not db_config: + raise D2DockerError("Invalid PostgreSQL URL format") + + dhis_conf_file = build_dhis_conf_from_external_db( + jdbc_url=db_config['url'], + username=db_config['user'], + password=db_config['password'] + ) + dhis_conf = dhis_conf_file.name + env_pairs = [ ("DHIS2_DATA_IMAGE", final_image_name), ("DHIS2_CORE_PORT", str(port)) if port else None, @@ -298,6 +315,8 @@ def run_docker_compose( ("ROOT_PATH", ROOT_PATH), ("PSQL_ENABLE_QUERY_LOGS", "") if not enable_postgres_queries_logging else None, ("EXTERNAL_DB_VOLUME", external_db_volume) if external_db_volume else None, + ("EXTERNAL_DB_URL", external_db_url) if external_db_url else None, + ("LOAD_DUMP_FROM_DATA", "yes" if load_dump_from_data else "no") if external_db_url else "no", ] env = dict((k, v) for (k, v) in [pair for pair in env_pairs if pair] if v is not None) @@ -314,6 +333,12 @@ def process_yaml(data): 'device': external_db_volume } } + if "EXTERNAL_DB_URL" in env: + del data["services"]["db"] + del data["volumes"]["pgdata"] + core = data["services"]["core"] + core["depends_on"] = [dep for dep in core["depends_on"] if dep != "db"] + core["extra_hosts"] = ["host.docker.internal:host-gateway"] return data @@ -339,6 +364,95 @@ def build_docker_compose(process_yaml): return temp_compose + +def validate_external_db_connection(db_url): + """Validate connection to external PostgreSQL database.""" + logger.info("Validating external database connection...") + try: + psql_cmd = ["psql", "-d", db_url, "-c", "SELECT now();", "&> /dev/null"] + run(psql_cmd, capture_output=False) + logger.info("External database validation successful") + except Exception as e: + raise D2DockerError(f"External database validation failed: {e}") + + +def parse_postgres_url(url: str) -> Optional[Dict[str, str]]: + """Parse PostgreSQL connection URL. + + Args: + url (str): PostgreSQL connection URL in the format postgresql://user:pass@host:port/dbname + + Raises: + D2DockerError: If the URL is invalid. + + Returns: + Optional[Dict[str, str]]: Dictionary with keys 'url', 'user', 'password' or None if url is empty. + """ + if not url: + return None + + try: + parsed = urlparse(url) + if parsed.scheme not in ['postgresql']: + raise D2DockerError(f"Invalid PostgreSQL URL scheme: {parsed.scheme}") + + if None in [parsed.username, parsed.password, parsed.hostname, parsed.path]: + raise D2DockerError(f"Missing components in PostgreSQL URL: {url}") + + if parsed.hostname == "localhost": + hostname = "host.docker.internal" + else: + hostname = parsed.hostname + + port = ":"+str(parsed.port) if parsed.port else "" + + return { + 'url': "jdbc:"+parsed.scheme+"://"+str(hostname)+port+parsed.path, + 'user': str(parsed.username), + 'password': str(parsed.password), + } + except Exception as e: + raise D2DockerError(f"Failed to parse PostgreSQL URL: {e}") + + +def build_dhis_conf_from_external_db(jdbc_url, username, password): + base_config_path = os.path.join(ROOT_PATH, "config", "DHIS2_home", "dhis.conf") + with open(base_config_path, 'r') as file: + config = file.read() + + # Update connection URL + config = re.sub( + r'^connection\.url\s*=.*$', + f'connection.url = {jdbc_url}', + config, + flags=re.MULTILINE + ) + + if username: + config = re.sub( + r'^connection\.username\s*=.*$', + f'connection.username = {username}', + config, + flags=re.MULTILINE + ) + + if password: + config = re.sub( + r'^connection\.password\s*=.*$', + f'connection.password = {password}', + config, + flags=re.MULTILINE + ) + + with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.conf') as temp_conf: + temp_conf.write(config) + + os.chmod(temp_conf.name, 0o644) + + atexit.register(lambda: os.remove(temp_conf.name)) + return temp_conf + + def get_config_path(default_filename, path): return os.path.abspath(path) if path else get_config_file(default_filename) @@ -474,6 +588,7 @@ def export_data_from_image(source_image, dest_path): # https://github.com/dhis2/dhis2-core/blob/master/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceDomain.java#L35 + default_folders = [ "apps", "dataValue", @@ -486,6 +601,7 @@ def export_data_from_image(source_image, dest_path): "jobData" ] + def export_data_from_running_containers(image_name, containers, destination, folders=None): """Export data (db + apps + documents) from a running Docker container to some folder.""" logger.info("Copy Dhis2 apps")