Feature: Add external database volume and external database connection

README.md

@@ -94,6 +94,9 @@ Some notes:
 - Use option `--java-opts="JAVA_OPTS"` to override the default JAVA_OPTS for the Tomcat process. That's tipically used to set the maximum/initial Heap Memory size (for example: `--java-opts="-Xmx3500m -Xms2500m"`)
 - Use option `--postgis-version=13-3.1-alpine` to specify the PostGIS version to use. By default, 10-2.5-alpine is used.
 - Use option `--debug-port=PORT` to specify the debug port of the Tomcat process.
+- Use option `--external-db-volume=VOLUME_ABSOLUTE_PATH` to create or use a persistant volume for the database located at `VOLUME_ABSOLUTE_PATH`.
+- Use option `--external-db-url=POSTGRES_URL` to connect to an external PostgreSQL database instead of using the DB container. The URL should be in the format `postgresql://USER:PASSWORD@HOST:PORT/DBNAME`. Note that you have configure the DB to accept connections from the docker internal network.
+- Use option `--load-dump-from-data` with `--external-db-url` to import the SQL dump to the external database. Equivalent to running without `-k`/`--keep-containers`. The recieving DB should have the appropiate config (DB owner, user permissions, postgis extention).
 
 #### Custom DHIS2 dhis.conf
 

src/d2_docker/commands/start.py

@@ -41,8 +41,23 @@ def setup(parser):
     parser.add_argument("--postgis-version", type=str, help="Set PostGIS database version")
     parser.add_argument("--enable-postgres-queries-logging", action="store_true",
                         help="Enable Postgres queries logging")
-
     parser.add_argument("--glowroot-port", metavar="PORT", help="Set glowroot port")
+    parser.add_argument(
+        "--external-db-volume",
+        metavar="DIRECTORY",
+        help="Directory for external database volume",
+    )
+    parser.add_argument(
+        "--external-db-url",
+        type=str,
+        metavar="postgresql://user:pass@host:port/dbname",
+        help="Use external PostgreSQL database"
+    )
+    parser.add_argument(
+        "--load-dump-from-data",
+        action="store_true",
+        help="Load database dump from data container (only with --external-db-url)",
+    )
 
 
 def run(args):
@@ -54,9 +69,25 @@ def run(args):
         image2 = args.image
 
     args.image = image2
+
+    if args.external_db_volume:
+        check_db_volume_path(args.external_db_volume)
+
+    if args.external_db_url:
+        utils.validate_external_db_connection(args.external_db_url)
+
     start(args)
 
 
+def check_db_volume_path(external_db_volume):
+    if not os.path.isabs(external_db_volume):
+        msg = "--external-db-volume must be an absolute path: {}".format(external_db_volume)
+        raise utils.D2DockerError(msg)
+    if not os.path.exists(external_db_volume):
+        msg = "--external-db-volume path does not exist: {}".format(external_db_volume)
+        raise utils.D2DockerError(msg)
+
+
 def import_from_file(images_path):
     dhis2_data_image_re = "/{}:".format(utils.DHIS2_DATA_IMAGE)
     result = utils.load_images_file(images_path)
@@ -85,10 +116,13 @@ def start(args):
     override_containers = not args.keep_containers
 
     if args.pull:
-        utils.run_docker_compose(["pull"], image_name, core_image=core_image)
+        utils.run_docker_compose(["pull"], image_name, core_image=core_image,
+                                 external_db_volume=args.external_db_volume)
 
     if override_containers:
-        utils.run_docker_compose(["down", "--volumes"], image_name, core_image=core_image)
+        utils.run_docker_compose(["down", "--volumes"], image_name, core_image=core_image,
+                                 external_db_volume=args.external_db_volume,
+                                 external_db_url=args.external_db_url)
 
     up_args = filter(
         bool, ["--force-recreate" if override_containers else None, "-d" if args.detach else None]
@@ -115,7 +149,10 @@ def start(args):
             java_opts=args.java_opts,
             postgis_version=args.postgis_version,
             enable_postgres_queries_logging=args.enable_postgres_queries_logging,
-            glowroot_port=args.glowroot_port
+            glowroot_port=args.glowroot_port,
+            external_db_volume=args.external_db_volume,
+            external_db_url=args.external_db_url,
+            load_dump_from_data=args.load_dump_from_data,
         )
 
     if args.detach:

src/d2_docker/config/dhis2-core-start.sh

@@ -10,17 +10,23 @@ set -e -u -o pipefail
 #
 
 # Global: LOAD_FROM_DATA="yes" | "no"
+# Global: LOAD_DUMP_FROM_DATA="yes" | "no"
+# Global: EXTERNAL_DB_URL=string (optional)
 # Global: DEPLOY_PATH=string
 # Global: DHIS2_AUTH=string
 
 export PGPASSWORD="dhis"
+db_url=""
+if [[ -n "$EXTERNAL_DB_URL" ]]; then
+    db_url="${EXTERNAL_DB_URL//localhost/host.docker.internal}"
+fi
 
 dhis2_url="http://localhost:8080/$DEPLOY_PATH"
 dhis2_url_with_auth="http://$DHIS2_AUTH@localhost:8080/$DEPLOY_PATH"
-psql_base_cmd="psql --quiet -h db -U dhis dhis2"
+psql_base_cmd="psql --quiet ${db_url:-"-h db -U dhis dhis2"}"
 psql_cmd="$psql_base_cmd -v ON_ERROR_STOP=0"
 psql_strict_cmd="$psql_base_cmd -v ON_ERROR_STOP=1"
-pgrestore_cmd="pg_restore -h db -U dhis -d dhis2"
+pgrestore_cmd="pg_restore ${db_url:-"-h db -U dhis dhis2"}"
 configdir="/config"
 homedir="/dhis2-home-files"
 scripts_dir="/data/scripts"
@@ -37,7 +43,12 @@ debug() {
 }
 
 run_sql_files() {
-    base_db_path=$(test "${LOAD_FROM_DATA}" = "yes" && echo "$root_db_path" || echo "$post_db_path")
+    if { [ -z "$db_url" ] && [ "${LOAD_FROM_DATA}" = "yes" ]; } ||
+        { [ -n "$db_url" ] && [ "${LOAD_FROM_DATA}" = "yes" ] && [ "${LOAD_DUMP_FROM_DATA}" = "yes" ]; }; then
+        base_db_path="$root_db_path"
+    else
+        base_db_path="$post_db_path"
+    fi
     debug "Files in data path"
     find "$base_db_path" >&2
 
@@ -69,10 +80,10 @@ run_psql_cmd() {
     local path=$1
     if [[ "$path" == *strict* ]]; then
         echo "Strict mode: $path"
-        $psql_strict_cmd < "$path"
+        $psql_strict_cmd <"$path"
     else
         echo "Normal mode: $path"
-        $psql_cmd < "$path"
+        $psql_cmd <"$path"
     fi
 }
 

src/d2_docker/docker-compose.yml

@@ -17,6 +17,8 @@ services:
             CATALINA_OPTS: "-Dcontext.path=${DEPLOY_PATH} -Xdebug -Xrunjdwp:transport=dt_socket,address=0.0.0.0:8000,server=y,suspend=n"
             JAVA_OPTS: "-Xmx7500m -Xms4000m ${JAVA_OPTS}"
             LOAD_FROM_DATA: "${LOAD_FROM_DATA}"
+            EXTERNAL_DB_URL: "${EXTERNAL_DB_URL}"
+            LOAD_DUMP_FROM_DATA: "${LOAD_DUMP_FROM_DATA}"
             DEPLOY_PATH: "${DEPLOY_PATH}"
             DHIS2_AUTH: "${DHIS2_AUTH}"
         entrypoint: bash /config/dhis2-core-entrypoint.sh

src/d2_docker/utils.py

@@ -10,9 +10,10 @@
 import time
 import yaml
 import urllib.request
+from urllib.parse import urlparse
 from setuptools._distutils import dir_util
 from pathlib import Path
-from typing import Optional
+from typing import Dict, Optional
 
 import d2_docker
 from d2_docker.glowroot import get_port_glowroot
@@ -25,6 +26,7 @@
 PROJECT_DIR = os.path.dirname(os.path.realpath(__file__))
 ROOT_PATH = os.environ.get("ROOT_PATH") or PROJECT_DIR
 
+
 def get_dhis2_war_url(version):
     match = (re.match(r"^(\d+.\d+)", version) if version.startswith("2.")
              else re.match(r"^(\d+)", version))
@@ -263,6 +265,9 @@ def run_docker_compose(
     postgis_version=None,
     enable_postgres_queries_logging=False,
     glowroot_port=None,
+    external_db_volume=None,
+    external_db_url=None,
+    load_dump_from_data=False,
     **kwargs,
 ):
     """
@@ -279,6 +284,19 @@ def run_docker_compose(
     post_sql_dir_abs = get_absdir_for_docker_volume(post_sql_dir)
     scripts_dir_abs = get_absdir_for_docker_volume(scripts_dir)
 
+    if external_db_url and not dhis_conf:
+        logger.info("External DB URL provided, updating dhis.conf")
+        db_config = parse_postgres_url(external_db_url)
+        if not db_config:
+            raise D2DockerError("Invalid PostgreSQL URL format")
+
+        dhis_conf_file = build_dhis_conf_from_external_db(
+            jdbc_url=db_config['url'],
+            username=db_config['user'],
+            password=db_config['password']
+        )
+        dhis_conf = dhis_conf_file.name
+
     env_pairs = [
         ("DHIS2_DATA_IMAGE", final_image_name),
         ("DHIS2_CORE_PORT", str(port)) if port else None,
@@ -300,6 +318,9 @@ def run_docker_compose(
         ("ROOT_PATH", ROOT_PATH),
         ("PSQL_ENABLE_QUERY_LOGS", "") if not enable_postgres_queries_logging else None,
         ("GLOWROOT_PORT", get_port_glowroot(glowroot_port))
+        ("EXTERNAL_DB_VOLUME", external_db_volume) if external_db_volume else None,
+        ("EXTERNAL_DB_URL", external_db_url) if external_db_url else None,
+        ("LOAD_DUMP_FROM_DATA", "yes" if load_dump_from_data else "no") if external_db_url else "no",
     ]
     env = dict((k, v) for (k, v) in [pair for pair in env_pairs if pair] if v is not None)
 
@@ -310,6 +331,21 @@ def process_yaml(data):
         for env_port in env_ports:
             if env_port not in env:
                 core["ports"] = [port for port in core["ports"] if env_port not in port]
+        if "EXTERNAL_DB_VOLUME" in env:
+            data["volumes"]["pgdata"] = {
+                'driver': 'local',
+                'driver_opts': {
+                    'type': 'none',
+                    'o': 'bind',
+                    'device': external_db_volume
+                }
+            }
+        if "EXTERNAL_DB_URL" in env:
+            del data["services"]["db"]
+            del data["volumes"]["pgdata"]
+            core = data["services"]["core"]
+            core["depends_on"] = [dep for dep in core["depends_on"] if dep != "db"]
+            core["extra_hosts"] = ["host.docker.internal:host-gateway"]
 
         return data
 
@@ -335,6 +371,95 @@ def build_docker_compose(process_yaml):
 
     return temp_compose
 
+
+def validate_external_db_connection(db_url):
+    """Validate connection to external PostgreSQL database."""
+    logger.info("Validating external database connection...")
+    try:
+        psql_cmd = ["psql", "-d", db_url, "-c", "SELECT now();", "&> /dev/null"]
+        run(psql_cmd, capture_output=False)
+        logger.info("External database validation successful")
+    except Exception as e:
+        raise D2DockerError(f"External database validation failed: {e}")
+
+
+def parse_postgres_url(url: str) -> Optional[Dict[str, str]]:
+    """Parse PostgreSQL connection URL.
+
+    Args:
+        url (str): PostgreSQL connection URL in the format postgresql://user:pass@host:port/dbname
+
+    Raises:
+        D2DockerError: If the URL is invalid.
+
+    Returns:
+        Optional[Dict[str, str]]: Dictionary with keys 'url', 'user', 'password' or None if url is empty.
+    """
+    if not url:
+        return None
+
+    try:
+        parsed = urlparse(url)
+        if parsed.scheme not in ['postgresql']:
+            raise D2DockerError(f"Invalid PostgreSQL URL scheme: {parsed.scheme}")
+
+        if None in [parsed.username, parsed.password, parsed.hostname, parsed.path]:
+            raise D2DockerError(f"Missing components in PostgreSQL URL: {url}")
+
+        if parsed.hostname == "localhost":
+            hostname = "host.docker.internal"
+        else:
+            hostname = parsed.hostname
+
+        port = ":"+str(parsed.port) if parsed.port else ""
+
+        return {
+            'url': "jdbc:"+parsed.scheme+"://"+str(hostname)+port+parsed.path,
+            'user': str(parsed.username),
+            'password': str(parsed.password),
+        }
+    except Exception as e:
+        raise D2DockerError(f"Failed to parse PostgreSQL URL: {e}")
+
+
+def build_dhis_conf_from_external_db(jdbc_url, username, password):
+    base_config_path = os.path.join(ROOT_PATH, "config", "DHIS2_home", "dhis.conf")
+    with open(base_config_path, 'r') as file:
+        config = file.read()
+
+    # Update connection URL
+    config = re.sub(
+        r'^connection\.url\s*=.*$',
+        f'connection.url = {jdbc_url}',
+        config,
+        flags=re.MULTILINE
+    )
+
+    if username:
+        config = re.sub(
+            r'^connection\.username\s*=.*$',
+            f'connection.username = {username}',
+            config,
+            flags=re.MULTILINE
+        )
+
+    if password:
+        config = re.sub(
+            r'^connection\.password\s*=.*$',
+            f'connection.password = {password}',
+            config,
+            flags=re.MULTILINE
+        )
+
+    with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.conf') as temp_conf:
+        temp_conf.write(config)
+
+    os.chmod(temp_conf.name, 0o644)
+
+    atexit.register(lambda: os.remove(temp_conf.name))
+    return temp_conf
+
+
 def get_config_path(default_filename, path):
     return os.path.abspath(path) if path else get_config_file(default_filename)
 
@@ -470,6 +595,7 @@ def export_data_from_image(source_image, dest_path):
 
 # https://github.com/dhis2/dhis2-core/blob/master/dhis-2/dhis-api/src/main/java/org/hisp/dhis/fileresource/FileResourceDomain.java#L35
 
+
 default_folders = [
     "apps",
     "dataValue",
@@ -482,6 +608,7 @@ def export_data_from_image(source_image, dest_path):
     "jobData"
 ]
 
+
 def export_data_from_running_containers(image_name, containers, destination, folders=None):
     """Export data (db + apps + documents) from a running Docker container to some folder."""
     logger.info("Copy Dhis2 apps")