diff --git a/README.md b/README.md index 0b1c2b8..d6cd1cd 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,8 @@ - Operating System: GNU/Linux or Windows 10. - Python >= 3.5 (with setuptools) -- Docker >= 18 -- Docker compose >= 1.17 +- Docker >= 20.10.13 +- Docker compose >= 2.0 - RAM memory: At least 4Gb for instance, preferrably 8Gb. On Ubuntu 22.04: @@ -69,6 +69,8 @@ Create a dhis2-data image from a .sql.gz SQL file and the apps and documents (or $ d2-docker create data docker.eyeseetea.com/eyeseetea/dhis2-data:2.37.9-sierra --sql=sierra-db.sql.gz [--apps-dir=path/to/apps] [--documents-dir=path/to/document] [--datavalues-dir=path/to/dataValue] ``` +There are demo database files at [databases.dhis2.org](https://databases.dhis2.org/) that may be used for testing purposses. The database downloaded should correspond to the core version created; if there is no database file for the created core version, a prior version of the database should work. + ### Start a DHIS2 instance Start a new container from a _dhis2-data_ base image: @@ -340,7 +342,8 @@ $ bash build-docker-container.sh ## Debug SQL queries -By default, d2-docker logs all SQL queries executed (one file per weekday). Example: +To enable SQL-query logging, start your instance with the --enable-postgres-queries-logging option. +d2-docker will log all SQL queries executed to a log named with the weekday. Example: ``` $ db_container="d2-docker-docker-eyeseetea-com-samaritans-40-4-0-sp-cpr-test-db-1" @@ -376,3 +379,18 @@ $ cp flaskenv.secret ~/.config/d2-docker/ $ curl -sS 'http://localhost:5000/harbor/https://docker.eyeseetea.com/api/v2.0/quotas/1' | jq ``` + +## Glowroot + +Glowroot is an open-source Java APM (Application Performance Monitoring) tool. It can help detect and diagnose application performance problems, tracing slow requests, errors, response time breakdowns, SQL capture and more. +When starting a container, there are two options to enable glowroot on the Tomcat process: +- Use option `--glowroot` to use the latest version of glowroot in the Tomcat process. This requires internet access to be able to retrieve the file. +- Use option `--glowroot-zip=FILE` to specify the zip file with the version of glowroot to run in the Tomcat process. This takes precedence over the other option. +When enabling glowroot, it will start listening on port 4000/tcp so you can connect via browser to its interface. You may override this port with: +- `--glowroot-port=PORT` to specify the APM glowroot port. + +### Run d2-docker with glowroot enabled in the default port at the latest version available + +``` +$ d2-docker start docker.eyeseetea.com/eyeseetea/dhis2-data:2.37.9-sierra --glowroot +``` diff --git a/requirements.txt b/requirements.txt index 246d3da..6dd5ad7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -Flask==3.1.0 +Flask==3.1.1 python-dotenv Flask_Cors==6.0.0 -requests==2.32.3 +requests==2.32.4 PyYAML setuptools diff --git a/setup.py b/setup.py index 86d1293..423e8ad 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,7 @@ setuptools.setup( name="d2_docker", - version="1.15.0.b2", + version="1.16.0", description="Dockers for DHIS2 instances", long_description=open("README.md", encoding="utf-8").read(), keywords=["python"], diff --git a/src/d2_docker/commands/rm.py b/src/d2_docker/commands/rm.py index a645f52..b619d26 100644 --- a/src/d2_docker/commands/rm.py +++ b/src/d2_docker/commands/rm.py @@ -15,7 +15,7 @@ def run(args): def remove_image(image): utils.logger.info("Delete image/containers: {}".format(image)) utils.run_docker_compose(["stop"], image) - result = utils.run_docker_compose(["ps", "-q"], data_image=image, capture_output=True) + result = utils.run_docker_compose(["ps", "-aq"], data_image=image, capture_output=True) container_ids = result.stdout.decode("utf-8").splitlines() utils.logger.debug("Container IDs: {}".format(container_ids)) if container_ids: diff --git a/src/d2_docker/commands/start.py b/src/d2_docker/commands/start.py index 2ff59f4..c59a5b8 100644 --- a/src/d2_docker/commands/start.py +++ b/src/d2_docker/commands/start.py @@ -41,6 +41,10 @@ def setup(parser): parser.add_argument("--postgis-version", type=str, help="Set PostGIS database version") parser.add_argument("--enable-postgres-queries-logging", action="store_true", help="Enable Postgres queries logging") + + parser.add_argument("--glowroot", action="store_true", help="Enables glowroot in tomcat in latest version") + parser.add_argument("--glowroot-zip", metavar="FILE", help="ZIP file with glowroot binaries") + parser.add_argument("--glowroot-port", metavar="PORT", help="Set glowroot port") def run(args): @@ -113,6 +117,9 @@ def start(args): java_opts=args.java_opts, postgis_version=args.postgis_version, enable_postgres_queries_logging=args.enable_postgres_queries_logging, + glowroot=args.glowroot, + glowroot_zip=args.glowroot_zip, + glowroot_port=args.glowroot_port, ) if args.detach: diff --git a/src/d2_docker/config/dhis2-core-entrypoint.sh b/src/d2_docker/config/dhis2-core-entrypoint.sh index 65610e4..ffc3a3f 100755 --- a/src/d2_docker/config/dhis2-core-entrypoint.sh +++ b/src/d2_docker/config/dhis2-core-entrypoint.sh @@ -12,6 +12,9 @@ WARFILE=/usr/local/tomcat/webapps/ROOT.war TOMCATDIR=/usr/local/tomcat DHIS2HOME=/DHIS2_home DATA_DIR=/data +GLOWROOT_ZIP="/opt/glowroot.zip" +GLOWROOT_DIR="/opt/glowroot" + debug() { echo "[dhis2-core-entrypoint] $*" >&2 @@ -43,12 +46,33 @@ wait_for_data_container_to_finish_copy() { } +setup_glowroot() { + if [ -f $GLOWROOT_ZIP ] && [ ! -d $GLOWROOT_DIR ] ; then + status=0 + output=$(unzip -q "$GLOWROOT_ZIP" -d /opt/ 2>&1) || status=$? + # Ignore RC=1 that implies only warnings and no errors (like an empty zip file) + if [ $status -gt 1 ]; then + echo "$output" + exit $status + fi + if [ -d $GLOWROOT_DIR ] ; then + echo '{ "web": { "bindAddress": "0.0.0.0", "port": "4000" }}' > $GLOWROOT_DIR/admin.json + chown -R tomcat:tomcat $GLOWROOT_DIR + chmod -R u=rwX,g=rX,o-rwx $GLOWROOT_DIR + echo 'export CATALINA_OPTS="$CATALINA_OPTS -javaagent:/opt/glowroot/glowroot.jar"' > /usr/local/tomcat/bin/setenv.sh + chmod ugo+x /usr/local/tomcat/bin/setenv.sh + chown tomcat:tomcat /usr/local/tomcat/bin/setenv.sh + fi + fi +} + if [ "$(id -u)" = "0" ]; then if [ -f $WARFILE ]; then unzip -q $WARFILE -d $TOMCATDIR/webapps/ROOT rm -v $WARFILE # just to save space fi + setup_glowroot wait_for_data_container_to_finish_copy mkdir -p $DATA_DIR/apps diff --git a/src/d2_docker/config/dhis2-core-start.sh b/src/d2_docker/config/dhis2-core-start.sh index 4a55c5b..508fcde 100755 --- a/src/d2_docker/config/dhis2-core-start.sh +++ b/src/d2_docker/config/dhis2-core-start.sh @@ -163,13 +163,13 @@ run() { local host=$1 psql_port=$2 setup_tomcat - copy_apps - copy_documents - copy_datavalues - if is_init_done; then - debug "Container: already configured. Skip DB load" + debug "Container: already configured. Skip DB load and keeping other changes" else + debug "Container: clean. Copying tomcat files and dhis folders" + copy_apps + copy_documents + copy_datavalues debug "Container: clean. Load DB" wait_for_postgres run_sql_files diff --git a/src/d2_docker/docker-compose.yml b/src/d2_docker/docker-compose.yml index bf6d447..1b8eda2 100644 --- a/src/d2_docker/docker-compose.yml +++ b/src/d2_docker/docker-compose.yml @@ -5,6 +5,7 @@ services: - "com.eyeseetea.image-name=${DHIS2_DATA_IMAGE}" volumes: - home:/DHIS2_home + - ${GLOWROOT_ZIP}:/opt/glowroot.zip - ${DHIS_CONF}:/config/override/dhis2/dhis.conf - ${ROOT_PATH}/config:/config - data:/data @@ -27,6 +28,8 @@ services: - "data" ports: - "${DHIS2_CORE_DEBUG_PORT}" + - "${GLOWROOT_PORT}" + dns_search: . db: image: "postgis/postgis:${POSTGIS_VERSION:-14-3.2-alpine}" shm_size: 1gb diff --git a/src/d2_docker/glowroot.py b/src/d2_docker/glowroot.py new file mode 100644 index 0000000..a5a542f --- /dev/null +++ b/src/d2_docker/glowroot.py @@ -0,0 +1,47 @@ +import atexit +import json +import os +import shutil +import tempfile +import urllib.request +import zipfile +from d2_docker import utils + +GLOWROOT_DEFAULT_PORT = "4000" + +def get_latest_glowroot_url(): + glowroot_releases_url = "https://api.github.com/repos/glowroot/glowroot/releases/latest" + glowroot_download_url = "https://github.com/glowroot/glowroot/releases/download" + with urllib.request.urlopen(glowroot_releases_url) as response: + data = response.read().decode() + release_info = json.loads(data) + + tag_name = release_info["tag_name"] + return "{}/{}/glowroot-{}-dist.zip".format(glowroot_download_url, tag_name, tag_name.lstrip("v")) + +def get_glowroot_zip(command, glowroot_zip, glowroot): + logger = utils.logger + if isinstance(command, list) and command[0] == "up": + glowroot_file = tempfile.NamedTemporaryFile(delete=False, prefix="glowroot_", suffix=".zip", dir="/tmp") + glowroot_path = glowroot_file.name + + atexit.register(lambda: os.remove(glowroot_path) if os.path.exists(glowroot_path) else None) + if glowroot_zip: + logger.debug("Copy zip file: {} -> {}".format(glowroot_zip, glowroot_path)) + shutil.copy(glowroot_zip, glowroot_path) + elif glowroot: + glowroot_url = get_latest_glowroot_url() + logger.info("Download file: {}".format(glowroot_url)) + urllib.request.urlretrieve(glowroot_url, glowroot_path) + else: + # empty zipfile + with zipfile.ZipFile(glowroot_path, mode="w") as zf: + pass + else: + glowroot_path = None + + return utils.get_absfile_for_docker_volume(glowroot_path) + +def get_port_glowroot(glowroot_port, glowroot_zip, glowroot): + port = glowroot_port if glowroot_port else GLOWROOT_DEFAULT_PORT + return "{}:{}".format(port, GLOWROOT_DEFAULT_PORT) if (glowroot_port or glowroot_zip or glowroot) else None diff --git a/src/d2_docker/utils.py b/src/d2_docker/utils.py index c92cec4..bc2bf93 100644 --- a/src/d2_docker/utils.py +++ b/src/d2_docker/utils.py @@ -15,6 +15,7 @@ from typing import Optional import d2_docker +from d2_docker.glowroot import get_glowroot_zip, get_port_glowroot from .image_name import ImageName PROJECT_NAME_PREFIX = "d2-docker" @@ -261,6 +262,9 @@ def run_docker_compose( tomcat_server=None, postgis_version=None, enable_postgres_queries_logging=False, + glowroot=None, + glowroot_zip=None, + glowroot_port=None, **kwargs, ): """ @@ -270,6 +274,7 @@ def run_docker_compose( The DHIS2_CORE_IMAGE is inferred from the data repo, if not specified. """ + final_image_name = data_image or get_running_image_name() project_name = get_project_name(final_image_name) core_image_name = core_image or get_core_image_name(data_image) @@ -296,13 +301,18 @@ def run_docker_compose( # Add ROOT_PATH from environment (required when run inside a docker) ("ROOT_PATH", ROOT_PATH), ("PSQL_ENABLE_QUERY_LOGS", "") if not enable_postgres_queries_logging else None, + ("GLOWROOT_PORT", get_port_glowroot(glowroot_port, glowroot_zip, glowroot)), + ("GLOWROOT_ZIP", get_glowroot_zip(args, glowroot_zip, glowroot)), ] env = dict((k, v) for (k, v) in [pair for pair in env_pairs if pair] if v is not None) def process_yaml(data): - if "DHIS2_CORE_DEBUG_PORT" not in env: - core = data["services"]["core"] - core["ports"] = [port for port in core["ports"] if "DHIS2_CORE_DEBUG_PORT" not in port] + # Removes ports for "core" service in docker-compose if the environmental variables are not established + core = data["services"]["core"] + env_ports = ["DHIS2_CORE_DEBUG_PORT", "GLOWROOT_PORT"] + for env_port in env_ports: + if env_port not in env: + core["ports"] = [port for port in core["ports"] if env_port not in port] return data