Release 1.16.0

README.md

@@ -2,8 +2,8 @@
 
 - Operating System: GNU/Linux or Windows 10.
 - Python >= 3.5 (with setuptools)
-- Docker >= 18
-- Docker compose >= 1.17
+- Docker >= 20.10.13
+- Docker compose >= 2.0
 - RAM memory: At least 4Gb for instance, preferrably 8Gb.
 
 On Ubuntu 22.04:
@@ -69,6 +69,8 @@ Create a dhis2-data image from a .sql.gz SQL file and the apps and documents (or
 $ d2-docker create data docker.eyeseetea.com/eyeseetea/dhis2-data:2.37.9-sierra --sql=sierra-db.sql.gz [--apps-dir=path/to/apps] [--documents-dir=path/to/document] [--datavalues-dir=path/to/dataValue]
 ```
 
+There are demo database files at [databases.dhis2.org](https://databases.dhis2.org/) that may be used for testing purposses. The database downloaded should correspond to the core version created; if there is no database file for the created core version, a prior version of the database should work.
+
 ### Start a DHIS2 instance
 
 Start a new container from a _dhis2-data_ base image:
@@ -340,7 +342,8 @@ $ bash build-docker-container.sh
 
 ## Debug SQL queries
 
-By default, d2-docker logs all SQL queries executed (one file per weekday). Example:
+To enable SQL-query logging, start your instance with the --enable-postgres-queries-logging option.
+d2-docker will log all SQL queries executed to a log named with the weekday. Example:
 
 ```
 $ db_container="d2-docker-docker-eyeseetea-com-samaritans-40-4-0-sp-cpr-test-db-1"
@@ -376,3 +379,18 @@ $ cp flaskenv.secret ~/.config/d2-docker/
 
 $ curl -sS 'http://localhost:5000/harbor/https://docker.eyeseetea.com/api/v2.0/quotas/1' | jq
 ```
+
+## Glowroot
+
+Glowroot is an open-source Java APM (Application Performance Monitoring) tool. It can help detect and diagnose application performance problems, tracing slow requests, errors, response time breakdowns, SQL capture and more.
+When starting a container, there are two options to enable glowroot on the Tomcat process:
+- Use option `--glowroot` to use the latest version of glowroot in the Tomcat process. This requires internet access to be able to retrieve the file.
+- Use option `--glowroot-zip=FILE` to specify the zip file with the version of glowroot to run in the Tomcat process. This takes precedence over the other option.
+When enabling glowroot, it will start listening on port 4000/tcp so you can connect via browser to its interface. You may override this port with:
+- `--glowroot-port=PORT` to specify the APM glowroot port.
+
+### Run d2-docker with glowroot enabled in the default port at the latest version available
+
+```
+$ d2-docker start docker.eyeseetea.com/eyeseetea/dhis2-data:2.37.9-sierra --glowroot
+```

requirements.txt

@@ -1,6 +1,6 @@
-Flask==3.1.0
+Flask==3.1.1
 python-dotenv
 Flask_Cors==6.0.0
-requests==2.32.3
+requests==2.32.4
 PyYAML
 setuptools

setup.py

@@ -4,7 +4,7 @@
 
 setuptools.setup(
     name="d2_docker",
-    version="1.15.0.b2",
+    version="1.16.0",
     description="Dockers for DHIS2 instances",
     long_description=open("README.md", encoding="utf-8").read(),
     keywords=["python"],

src/d2_docker/commands/rm.py

@@ -15,7 +15,7 @@ def run(args):
 def remove_image(image):
     utils.logger.info("Delete image/containers: {}".format(image))
     utils.run_docker_compose(["stop"], image)
-    result = utils.run_docker_compose(["ps", "-q"], data_image=image, capture_output=True)
+    result = utils.run_docker_compose(["ps", "-aq"], data_image=image, capture_output=True)
     container_ids = result.stdout.decode("utf-8").splitlines()
     utils.logger.debug("Container IDs: {}".format(container_ids))
     if container_ids:

src/d2_docker/commands/start.py

@@ -41,6 +41,10 @@ def setup(parser):
     parser.add_argument("--postgis-version", type=str, help="Set PostGIS database version")
     parser.add_argument("--enable-postgres-queries-logging", action="store_true",
                         help="Enable Postgres queries logging")
+
+    parser.add_argument("--glowroot", action="store_true", help="Enables glowroot in tomcat in latest version")
+    parser.add_argument("--glowroot-zip", metavar="FILE", help="ZIP file with glowroot binaries")
+    parser.add_argument("--glowroot-port", metavar="PORT", help="Set glowroot port")
 
 
 def run(args):
@@ -113,6 +117,9 @@ def start(args):
             java_opts=args.java_opts,
             postgis_version=args.postgis_version,
             enable_postgres_queries_logging=args.enable_postgres_queries_logging,
+            glowroot=args.glowroot,
+            glowroot_zip=args.glowroot_zip,
+            glowroot_port=args.glowroot_port,
         )
 
     if args.detach:

src/d2_docker/config/dhis2-core-entrypoint.sh

@@ -12,6 +12,9 @@ WARFILE=/usr/local/tomcat/webapps/ROOT.war
 TOMCATDIR=/usr/local/tomcat
 DHIS2HOME=/DHIS2_home
 DATA_DIR=/data
+GLOWROOT_ZIP="/opt/glowroot.zip"
+GLOWROOT_DIR="/opt/glowroot"
+
 
 debug() {
     echo "[dhis2-core-entrypoint] $*" >&2
@@ -43,12 +46,33 @@ wait_for_data_container_to_finish_copy() {
 
 }
 
+setup_glowroot() {
+    if [ -f $GLOWROOT_ZIP ] && [ ! -d $GLOWROOT_DIR ] ; then
+        status=0
+        output=$(unzip -q "$GLOWROOT_ZIP" -d /opt/ 2>&1) || status=$?
+        # Ignore RC=1 that implies only warnings and no errors (like an empty zip file)
+        if [ $status -gt 1 ]; then
+            echo "$output"
+            exit $status
+        fi
+        if [ -d $GLOWROOT_DIR ] ; then
+            echo '{ "web": { "bindAddress": "0.0.0.0", "port": "4000" }}' > $GLOWROOT_DIR/admin.json
+            chown -R tomcat:tomcat $GLOWROOT_DIR
+            chmod -R u=rwX,g=rX,o-rwx $GLOWROOT_DIR
+            echo 'export CATALINA_OPTS="$CATALINA_OPTS -javaagent:/opt/glowroot/glowroot.jar"' > /usr/local/tomcat/bin/setenv.sh
+            chmod ugo+x /usr/local/tomcat/bin/setenv.sh
+            chown tomcat:tomcat /usr/local/tomcat/bin/setenv.sh
+        fi
+    fi
+}
+
 if [ "$(id -u)" = "0" ]; then
     if [ -f $WARFILE ]; then
         unzip -q $WARFILE -d $TOMCATDIR/webapps/ROOT
         rm -v $WARFILE # just to save space
     fi
 
+    setup_glowroot
     wait_for_data_container_to_finish_copy
 
     mkdir -p $DATA_DIR/apps

src/d2_docker/config/dhis2-core-start.sh

@@ -163,13 +163,13 @@ run() {
     local host=$1 psql_port=$2
 
     setup_tomcat
-    copy_apps
-    copy_documents
-    copy_datavalues
-
     if is_init_done; then
-        debug "Container: already configured. Skip DB load"
+        debug "Container: already configured. Skip DB load and keeping other changes"
     else
+        debug "Container: clean. Copying tomcat files and dhis folders"
+        copy_apps
+        copy_documents
+        copy_datavalues
         debug "Container: clean. Load DB"
         wait_for_postgres
         run_sql_files

src/d2_docker/docker-compose.yml

@@ -5,6 +5,7 @@ services:
             - "com.eyeseetea.image-name=${DHIS2_DATA_IMAGE}"
         volumes:
             - home:/DHIS2_home
+            - ${GLOWROOT_ZIP}:/opt/glowroot.zip
             - ${DHIS_CONF}:/config/override/dhis2/dhis.conf
             - ${ROOT_PATH}/config:/config
             - data:/data
@@ -27,6 +28,8 @@ services:
             - "data"
         ports:
             - "${DHIS2_CORE_DEBUG_PORT}"
+            - "${GLOWROOT_PORT}"
+        dns_search: .
     db:
         image: "postgis/postgis:${POSTGIS_VERSION:-14-3.2-alpine}"
         shm_size: 1gb

src/d2_docker/glowroot.py

@@ -0,0 +1,47 @@
+import atexit
+import json
+import os
+import shutil
+import tempfile
+import urllib.request
+import zipfile
+from d2_docker import utils
+
+GLOWROOT_DEFAULT_PORT = "4000"
+
+def get_latest_glowroot_url():
+    glowroot_releases_url = "https://api.github.com/repos/glowroot/glowroot/releases/latest"
+    glowroot_download_url = "https://github.com/glowroot/glowroot/releases/download"
+    with urllib.request.urlopen(glowroot_releases_url) as response:
+        data = response.read().decode()
+        release_info = json.loads(data)
+
+    tag_name = release_info["tag_name"]
+    return "{}/{}/glowroot-{}-dist.zip".format(glowroot_download_url, tag_name, tag_name.lstrip("v"))
+
+def get_glowroot_zip(command, glowroot_zip, glowroot):
+    logger = utils.logger
+    if isinstance(command, list) and command[0] == "up":
+        glowroot_file = tempfile.NamedTemporaryFile(delete=False, prefix="glowroot_", suffix=".zip", dir="/tmp")
+        glowroot_path = glowroot_file.name
+
+        atexit.register(lambda: os.remove(glowroot_path) if os.path.exists(glowroot_path) else None)
+        if glowroot_zip:
+            logger.debug("Copy zip file: {} -> {}".format(glowroot_zip, glowroot_path))
+            shutil.copy(glowroot_zip, glowroot_path)
+        elif glowroot:
+            glowroot_url = get_latest_glowroot_url()
+            logger.info("Download file: {}".format(glowroot_url))
+            urllib.request.urlretrieve(glowroot_url, glowroot_path)
+        else:
+            # empty zipfile
+            with zipfile.ZipFile(glowroot_path, mode="w") as zf:
+                pass
+    else:
+        glowroot_path = None
+
+    return utils.get_absfile_for_docker_volume(glowroot_path)
+
+def get_port_glowroot(glowroot_port, glowroot_zip, glowroot):
+    port = glowroot_port if glowroot_port else GLOWROOT_DEFAULT_PORT
+    return "{}:{}".format(port, GLOWROOT_DEFAULT_PORT) if (glowroot_port or glowroot_zip or glowroot) else None

src/d2_docker/utils.py

@@ -15,6 +15,7 @@
 from typing import Optional
 
 import d2_docker
+from d2_docker.glowroot import get_glowroot_zip, get_port_glowroot
 from .image_name import ImageName
 
 PROJECT_NAME_PREFIX = "d2-docker"
@@ -261,6 +262,9 @@ def run_docker_compose(
     tomcat_server=None,
     postgis_version=None,
     enable_postgres_queries_logging=False,
+    glowroot=None,
+    glowroot_zip=None,
+    glowroot_port=None,
     **kwargs,
 ):
     """
@@ -270,6 +274,7 @@ def run_docker_compose(
 
     The DHIS2_CORE_IMAGE is inferred from the data repo, if not specified.
     """
+
     final_image_name = data_image or get_running_image_name()
     project_name = get_project_name(final_image_name)
     core_image_name = core_image or get_core_image_name(data_image)
@@ -296,13 +301,18 @@ def run_docker_compose(
         # Add ROOT_PATH from environment (required when run inside a docker)
         ("ROOT_PATH", ROOT_PATH),
         ("PSQL_ENABLE_QUERY_LOGS", "") if not enable_postgres_queries_logging else None,
+        ("GLOWROOT_PORT", get_port_glowroot(glowroot_port, glowroot_zip, glowroot)),
+        ("GLOWROOT_ZIP", get_glowroot_zip(args, glowroot_zip, glowroot)),
     ]
     env = dict((k, v) for (k, v) in [pair for pair in env_pairs if pair] if v is not None)
 
     def process_yaml(data):
-        if "DHIS2_CORE_DEBUG_PORT" not in env:
-            core = data["services"]["core"]
-            core["ports"] = [port for port in core["ports"] if "DHIS2_CORE_DEBUG_PORT" not in port]
+        # Removes ports for "core" service in docker-compose if the environmental variables are not established
+        core = data["services"]["core"]
+        env_ports = ["DHIS2_CORE_DEBUG_PORT", "GLOWROOT_PORT"]
+        for env_port in env_ports:
+            if env_port not in env:
+                core["ports"] = [port for port in core["ports"] if env_port not in port]
 
         return data