hyraxctl

#!/bin/bash
#
# hyraxctl
# Can be used to start and stop the docker containers running Hyrax and the
# # Apache http services, either individually or together.
# In this implementation Apache httpd connects to Hyrax using AJP on port 8009.
#
# The functions are simple. Read the usage statement, and then read the code.
#
#

#########################################################################
# loggy()
#
# A handy logging function.
#
function loggy() {
    if test -n "$log_file"; then
        echo "$@" | awk '{ print "# "$0;}' | tee -a "$log_file" >&2
    else
        echo "$@" | awk '{ print "# "$0;}' >&2
    fi
}

export H1="######################################################################"
export H2="- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -"
export H3="-  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -  -"

export HYRAX_NAME="${HYRAX_NAME:-"hyrax"}"
export HTTPD_NAME="${HTTPD_NAME:-"httpd"}"
export HYRAX_HOME="${HYRAX_HOME:-"/home/ubuntu/hyrax"}"
export DATA_MOUNT="${DATA_MOUNT:-"/usr/share/hyrax"}"
export HYRAX_IMAGE="${HYRAX_IMAGE:-"opendap/hyrax:snapshot-el8"}"

export HTTPD_IMAGE_DEFAULT="httpd:latest"

# As of 11/16/203 we rolled back to this because of some issue we don't
# understand in httpd:latest - ndp 11/16/2023
export HTTPD_IMAGE_DEFAULT="httpd:2.4.56-bullseye"
export HTTPD_IMAGE="${HTTPD_IMAGE:-"$HTTPD_IMAGE_DEFAULT"}"

export APACHE_HTTPD_HOME="${APACHE_HTTPD_HOME:-"/usr/local/apache2"}"
export HYRAX_NETWORK="${HYRAX_NETWORK:-"hyrax-net"}"
export HYRAX_DEPLOYMENT="http://test.opendap.org"

export THREDDS_CATALOG_ROOT="${THREDDS_CATALOG_ROOT:-""}"

export HYRAX_MEMORY="${HYRAX_MEMORY:-"24g"}"
export RPM_BUILD_REPO="s3://opendap.travis.build"

export bes_cores_dir
export tomcat_server_xml
export run_privileged
export log_file

################################################################################
#
# get_recent_docker_image_tag_names()
#
# Here is a docker hub response for a tg name.
# DockerHub Tag Record 01/28/25
#     {
#      "creator": 13215106,
#      "id": 1060286216,
#      "images": [
#        {
#          "architecture": "amd64",
#          "features": "",
#          "variant": null,
#          "digest": "sha256:d4145a8e5a1465f30ce46a3a26104d8f5580b6aa9a8e9beb207c596b6af9f5af",
#          "os": "linux",
#          "os_features": "",
#          "os_version": null,
#          "size": 1283353276,
#          "status": "active",
#          "last_pulled": "2026-01-28T18:29:29.35703527Z",
#          "last_pushed": "2026-01-28T17:55:14.14516051Z"
#        }
#      ],
#      "last_updated": "2026-01-28T17:55:17.788855Z",
#      "last_updater": 13215106,
#      "last_updater_username": "travis4opendap",
#      "name": "ngap-1.17.1-665-test-deploy",
#      "repository": 1748063,
#      "full_size": 1283353276,
#      "v2": true,
#      "tag_status": "active",
#      "tag_last_pulled": "2026-01-28T18:29:29.35703527Z",
#      "tag_last_pushed": "2026-01-28T17:55:17.788855Z",
#      "media_type": "application/vnd.docker.container.image.v1+json",
#      "content_type": "image",
#      "digest": "sha256:d4145a8e5a1465f30ce46a3a26104d8f5580b6aa9a8e9beb207c596b6af9f5af"
#    },
#
function get_recent_docker_image_tag_names() {
    local prolog="get_recent_docker_image_tag_names() -"
    loggy "$prolog BEGIN"

    local dockerhub_ns="${1:-"opendap"}"
    loggy "$prolog     dockerhub_ns: $dockerhub_ns"

    local dockerhub_repo="${2:-"hyrax"}"
    loggy "$prolog   dockerhub_repo: $dockerhub_repo"

    local recent_tag_names
    recent_tag_names="$(curl -s "https://hub.docker.com/v2/namespaces/$dockerhub_ns/repositories/$dockerhub_repo/tags" | jq -r '.results[].name')"

    loggy "$prolog recent_tag_names: "
    loggy "$recent_tag_names"

    local most_recent_tag_name
    most_recent_tag_name="$(echo "$recent_tag_names" | head -n 1)"
    loggy echo "$prolog most_recent_tag_name: $most_recent_tag_name"
    loggy "$prolog END"
}

################################################################################
#
# show_config()
#
function show_config() {
    local prolog="show_config() -"
    loggy "$prolog $H2"
    loggy "$prolog Configuration Statement"
    loggy "$prolog "
    loggy "$prolog-- Primary Values: "
    loggy "$prolog"
    loggy "$prolog             HYRAX_NAME: $HYRAX_NAME"
    loggy "$prolog             HTTPD_NAME: $HTTPD_NAME"
    loggy "$prolog             HYRAX_HOME: $HYRAX_HOME"
    loggy "$prolog             DATA_MOUNT: $DATA_MOUNT"
    loggy "$prolog            HYRAX_IMAGE: $HYRAX_IMAGE"
    loggy "$prolog            HTTPD_IMAGE: $HTTPD_IMAGE"
    loggy "$prolog      APACHE_HTTPD_HOME: $APACHE_HTTPD_HOME"
    loggy "$prolog          HYRAX_NETWORK: $HYRAX_NETWORK"
    loggy "$prolog       HYRAX_DEPLOYMENT: $HYRAX_DEPLOYMENT"
    loggy "$prolog           HYRAX_MEMORY: $HYRAX_MEMORY"
    loggy "$prolog"
    loggy "$prolog-- Values derived from primary values: "
    loggy "$prolog"
    loggy "$prolog                  log_dir: $log_dir"
    loggy "$prolog                 data_dir: $data_dir"
    loggy "$prolog       bes_site_conf_file: $bes_site_conf_file"
    loggy "$prolog      bes_creds_conf_file: $bes_creds_conf_file"
    loggy "$prolog              bes_log_dir: $bes_log_dir"
    loggy "$prolog            bes_cores_dir: $bes_cores_dir"
    loggy "$prolog            olfs_conf_dir: $olfs_conf_dir"
    loggy "$prolog        tomcat_server_xml: $tomcat_server_xml"
    loggy "$prolog tomcat_server_xml_volume: $tomcat_server_xml_volume"
    loggy "$prolog               httpd_conf: $httpd_conf"
    loggy "$prolog           htdocs_content: $htdocs_content"
    loggy "$prolog               expires_sh: $expires_sh"
    loggy "$prolog                    ncwms: $ncwms"
    loggy "$prolog"
    loggy "$prolog $H2"
}
################################################################################
#
# config_tomcat_server()
#
function config_tomcat_server() {
    local hyrax_home="$1"
    # A Tomcat server.xml file that enables AJP pn 8009
    tomcat_server_xml="$hyrax_home/tomcat/server.xml"

    # Creates the tomcat server.xml volume mount key pair for the Docker run command.
    export tomcat_server_xml_volume=""
    if test -n "$tomcat_server_xml"; then
        export tomcat_server_xml_volume="--volume $tomcat_server_xml:/tomcat/conf/server.xml"
    fi

}

################################################################################
#
# config_test_opendap_org()
#
# shellcheck disable=SC2120
function config_test_opendap_org() {
    local prolog="config_test_opendap_org() -"
    local hyrax_home=${1:-"$HYRAX_HOME"}
    local data_dir_in=${2:-"$DATA_MOUNT"}

    loggy "$prolog $H1"
    loggy "$prolog BEGIN"
    loggy "$prolog "
    loggy "$prolog        Using hyrax_home: $hyrax_home"

    # Location of the log directories for bes, olfs**, and tomcat
    export log_dir="$hyrax_home/log"

    # Set up the location of the data directory for Hyrax.
    # If one is not provided the default "$hyrax_home/data"
    # will be used.
    if test -n "$data_dir_in"; then
        export data_dir="$data_dir_in"
    else
        export data_dir="$hyrax_home/data"
        loggy "$prolog Using default data_dir value of: $data_dir"
    fi

    # The location of the OLFS persistent configuration
    # to be injected into the Hyrax container at /etc/olfs
    #
    # **If this is used then the OLFS logs will be written
    # to "$hyrax_home/olfs" and the log directory
    # "$hyrax_home/log/olfs" will be ignored.
    export olfs_conf_dir="$hyrax_home/olfs"

    export bes_site_conf_file="$hyrax_home/bes/site.conf"

    # Credentials for service-chain
    export bes_creds_conf_file="$hyrax_home/bes/creds.conf"

    export bes_log_dir="$log_dir/bes"
    bes_cores_dir="$hyrax_home/bes/cores"

    config_tomcat_server "$hyrax_home"

    #
    # Variables for configuring Docker launch of httpd
    #

    # We inject a modified httpd.conf file
    export httpd_conf="$hyrax_home/httpd/httpd.conf"

    # Our own collection of things that need to be served via http,
    # possibly utilizing range get capabilities
    export htdocs_content="$hyrax_home/httpd/htdocs"

    # Q: Is expires.sh still in use? Can we drop it?
    # A: Yes it's still in use by libdap4 tests.
    # Q: Can we drop it?
    # A: No we can't without fixing various things
    export expires_sh="$hyrax_home/httpd/cgi-bin/expires.sh"

    # ncWMS host name to be injected into the ncWMS configuration.
    export ncwms="$HYRAX_DEPLOYMENT"

    show_config

    loggy "$prolog END"

}

################################################################################
#
# config_ndp_local()
#
function config_ndp_local() {
    local prolog="config_ndp_local() -"
    export HYRAX_HOME="/Users/ndp/OPeNDAP/hyrax/test.opendap.org"
    config_ndp "$HYRAX_HOME"
}

function config_ndp() {
    local prolog="config_ndp() -"
    local hyrax_home=
    hyrax_home="${1:-$HYRAX_HOME}"
    loggy "hyrax_home: $hyrax_home"
    local pfix
    prefix="${prefix:-"/Users/ndp/OPeNDAP/hyrax/build"}"
    export data_dir="$prefix/share/hyrax"

    loggy "$prolog $H1"
    loggy "$prolog BEGIN"
    loggy "$prolog "
    export ncwms="http://localhost:8080"
    export log_dir="$hyrax_home/log"
    export bes_log_dir="$log_dir/bes"
    export bes_cores_dir="$hyrax_home/bes/cores"
    export olfs_conf_dir="$hyrax_home/etc/olfs"
    export bes_site_conf_file="$hyrax_home/bes/site.conf"

    # Credentials for service-chain
    export bes_creds_conf_file="$hyrax_home/bes/creds.conf"

    export httpd_conf="$hyrax_home/httpd/httpd.conf"
    export htdocs_content="$hyrax_home/httpd/htdocs"
    export expires_sh="$hyrax_home/httpd/cgi-bin/expires.sh"

    config_tomcat_server "$hyrax_home"

    show_config

    loggy "$prolog END"

}
################################################################################
#
# config_rocky_8_local()
#
function config_rocky_8_local() {
    local prolog="config_rocky_8_local() -"
    export HYRAX_HOME="/home/rocky/docker_deploy"
    config_rocky_8 $HYRAX_HOME
}

function config_rocky_8() {
    local prolog="config_rocky_8() -"

    loggy "$prolog $H1"
    loggy "$prolog BEGIN"
    loggy "$prolog "

    local hyrax_home
    hyrax_home="${1:-$HYRAX_HOME}"

    local pfix
    prefix=${prefix:-"/home/rocky/hyrax/build"}
    export data_dir="$prefix/share/hyrax"

    export ncwms="http://localhost:8080"
    export log_dir="$hyrax_home/log"
    export bes_log_dir="$log_dir/bes"
    export bes_cores_dir="$hyrax_home/bes/cores"
    export olfs_conf_dir="/etc/olfs"
    export bes_site_conf_file="$hyrax_home/bes/site.conf"

    export httpd_conf="$hyrax_home/httpd/httpd.conf"
    export htdocs_content="$hyrax_home/httpd/htdocs"
    export expires_sh="$hyrax_home/httpd/cgi-bin/expires.sh"

    config_tomcat_server "$hyrax_home"

    show_config

    loggy "$prolog END"

}

################################################################################
#
# make_network_as_needed()
#
function make_network_as_needed() {
    local prolog="make_network_as_needed() -"
    #
    # Check for the network that will be shared with Apache httpd
    # If it's not found, create it.
    #
    local status=
    docker network inspect "$HYRAX_NETWORK" >/dev/null 2>&1
    status=$?
    if test $status -ne 0; then
        loggy "$prolog No Docker network by the name of $HYRAX_NETWORK was found. Creating..."
        docker network create "$HYRAX_NETWORK"
        loggy "$prolog Created Docker network '$HYRAX_NETWORK' for Hyrax and Apache httpd."
        loggy "$prolog "
    else
        loggy "$prolog Docker network '$HYRAX_NETWORK' exists, using."
        loggy "$prolog "
    fi

}

################################################################################
#
# delete_network()
#
function delete_network() {
    # Dump Hyrax Docker Network
    docker network rm "$HYRAX_NETWORK"
}

################################################################################
#
# start_httpd()
#
function start_httpd() {
    local prolog="start_httpd() -"

    make_network_as_needed

    loggy "$prolog Starting Docker container for Apache httpd: $HTTPD_IMAGE"
    docker run -d \
        --name "$HTTPD_NAME" \
        --net="$HYRAX_NETWORK" \
        --publish 80:80 \
        --publish 443:443 \
        --volume "$httpd_conf":"$APACHE_HTTPD_HOME/conf/httpd.conf" \
        --volume "$htdocs_content":"$APACHE_HTTPD_HOME/htdocs" \
        --volume "$expires_sh":"$APACHE_HTTPD_HOME/cgi-bin/expires.sh" \
        "$HTTPD_IMAGE"

    loggy "$prolog "
    loggy "$prolog Docker container: $HTTPD_IMAGE STARTED."
    loggy "$prolog END"

}

################################################################################
#
# stop_httpd()
#
function stop_httpd() {
    # Dump httpd container
    docker rm -f "$HTTPD_NAME"
}

################################################################################
#
# update_httpd()
#
function update_httpd() {
    docker pull "$HTTPD_IMAGE"
    stop_httpd
    start_httpd
}

################################################################################
#
# get_platform()
#
function get_platform() {
    local prolog="get_platform() -"
    local sys_arch=
    sys_arch=$(arch)
    loggy "$prolog sys_arch: $sys_arch"
    local platform=""
    if test $sys_arch = "arm64"; then platform="--platform linux/amd64"; fi
    echo "$platform"
}

################################################################################
#
# set_file_permissions()
#
function set_file_permissions() {
    local prolog="set_file_permissions() -"
    loggy "$prolog BEGIN"

    local vers_doc
    vers_doc=$(which sw_vers)
    if test $? -ne 0; then
        local name="testy_besty"

        local platform=""
        platform="$(get_platform)"
        loggy "$prolog platform: $platform"

        local use_disconnect="-d" #
        loggy "$prolog use_disconnect: $use_disconnect"

        loggy "$prolog Checking bes user info in Hyrax container: $HYRAX_IMAGE"
        docker run $use_disconnect $platform --name "$name" "$HYRAX_IMAGE"

        local sleep_time=10
        loggy "$prolog Waiting $sleep_time seconds for hyrax to get going..."
        sleep $sleep_time

        local bes_uid
        bes_uid=$(docker logs "$name" 2>&1 | grep bes_uid | awk '{print $2;}')
        loggy "$prolog bes_uid: $bes_uid"

        local bes_gid
        bes_gid=$(docker logs "$name" 2>&1 | grep bes_gid | awk '{print $2;}')
        loggy "$prolog bes_gid: $bes_gid"

        loggy "$prolog Stopping Hyrax container: $HYRAX_IMAGE"
        docker container rm -f "$name"

        if test -n "$bes_uid" && test -n "$bes_gid"; then
            loggy "$prolog Setting ownership of $bes_log_dir"
            sudo chown -R "$bes_uid":"$bes_gid" "$bes_log_dir"
        else
            loggy "$prolog ERROR - Failed to determine one or both of bes user uid or gid."
            return 1
        fi
    else
        loggy "$prolog Detected OSX, no permissions were changed."
        loggy "$(sw_vers)"
    fi
    loggy "$prolog END"
    return 0
}

################################################################################
#
# clean_hyrax_logs()
#
function clean_hyrax_logs() {
    local prolog="clean_hyrax_logs() -"
    loggy "$prolog BEGIN"
    loggy "$(rm -vrf "$HYRAX_HOME"/log/bes/*)"
    loggy "$(rm -vrf "$HYRAX_HOME"/olfs/logs/*)"
    loggy "$(rm -vrf "$HYRAX_HOME"/log/tomcat/*)"
    loggy "$prolog END"
}

################################################################################
# save_hyrax_logs()
# Saves the bes log file. The tomcat logs and OLFS logs are already being
# saved/rolled like a server should.
function save_hyrax_logs() {
    local prolog="save_hyrax_logs() -"
    loggy "$prolog BEGIN"

    local date_tag=
    date_tag=$(date +"%Y-%m-%d-%H:%M:%S")
    loggy "$prolog date_tag: $date_tag"

    local target_dir="$HYRAX_HOME/crash_logs/$date_tag"
    loggy "$prolog target_dir: $target_dir"

    mkdir -pv "$target_dir"

    loggy "$prolog Saving BES log..."
    local source_file=
    source_file="$HYRAX_HOME/log/bes/bes.log"
    loggy "$prolog Processing: $source_file"
    if test -f "$source_file"; then
        cp -v "$source_file" "$target_dir/bes.log"
    else
        loggy "$prolog Failed to locate: $source_file"
    fi

    loggy "$prolog Save the various OLFS logs..."
    for log in olfs-log.json HyraxAccess.log HyraxErrors.log cloud_watch_request.log cloud_watch_response.log BESCommands.log; do
        source_file="$HYRAX_HOME/olfs/logs/$log"
        loggy "$prolog Processing: $source_file"
        if test -f "$source_file"; then
            cp -v "$source_file" "$target_dir/$log"
        else
            loggy "$prolog Failed to locate: $source_file"
        fi
    done

    loggy "$prolog Save the Tomcat logs..."
    local local_host_log=
    local_host_log="$(basename "$(ls -1t "$HYRAX_HOME/log/tomcat/localhost".* | head -1)")"
    loggy "$prolog local_host_log: $local_host_log"
    for log in console.log "$local_host_log" catalina.out; do
        source_file="$HYRAX_HOME/log/tomcat/$log"
        loggy "$prolog Processing: $source_file"

        if test -f "$source_file"; then
            cp -v "$source_file" "$target_dir/tomcat-$log"
        else
            loggy "$prolog Failed to locate file: $source_file"
        fi
    done
    loggy "$prolog END"
}
################################################################################
#
# start_simple()
#
function start_debug() {
    local prolog="start_hyrax() -"
    loggy "$prolog BEGIN"

    save_hyrax_logs
    clean_hyrax_logs

    set_file_permissions
    if test $? -ne 0; then
        loggy "$prolog ERROR - Unable to set host file permissions. Hyrax was not started."
        return 1
    fi

    local use_disconnect="${disconnect:-"-d"}"
    loggy "$prolog use_disconnect: $use_disconnect"

    local platform=""
    platform="$(get_platform)"
    loggy "$prolog platform: $platform"

    loggy "$prolog run_privileged: $run_privileged"

    loggy "$prolog Starting Hyrax container: $HYRAX_IMAGE"
    docker run -d --name "$HYRAX_NAME" -p 8080:8080 -v /usr/share/hyrax:/usr/share/hyrax \
        -v /home/rocky/test.opendap.org/debug:/debug \
        "$HYRAX_IMAGE"
    loggy "$prolog"
    loggy "$prolog Docker container: $HYRAX_IMAGE STARTED."
    loggy "$prolog"
    loggy "$prolog END"

}

################################################################################
#
# start_hyrax()
#
function start_hyrax() {
    local prolog="start_hyrax() -"
    loggy "$prolog BEGIN"

    save_hyrax_logs
    clean_hyrax_logs

    set_file_permissions
    if test $? -ne 0; then
        loggy "$prolog ERROR - Unable to set host file permissions. Hyrax was not started."
        return 1
    fi

    make_network_as_needed

    local use_disconnect="${disconnect:-"-d"}"
    loggy "$prolog use_disconnect: $use_disconnect"

    local platform=""
    platform="$(get_platform)"
    loggy "$prolog platform: $platform"

    loggy "$prolog run_privileged: $run_privileged"

    loggy "$prolog Starting Hyrax container: $HYRAX_IMAGE"
    docker run $use_disconnect $run_privileged $platform \
        --name "$HYRAX_NAME" \
        --net="$HYRAX_NETWORK" \
        --publish 8009:8009 \
        --publish 8080:8080 \
        --publish 8443:8443 \
        --volume "$data_dir":/usr/share/hyrax:ro \
        --volume "$bes_log_dir":/var/log/bes \
        --volume "$bes_cores_dir":/cores \
        --volume "$log_dir/tomcat":/usr/share/tomcat/logs \
        --volume "$olfs_conf_dir":/etc/olfs \
        --volume "$bes_site_conf_file":/etc/bes/site.conf \
        --volume "$bes_creds_conf_file":/etc/bes/credentials.conf \
        --memory="$HYRAX_MEMORY" \
        $tomcat_server_xml_volume \
        "$HYRAX_IMAGE" \
        -e support@opendap.org \
        -s \
        -n $ncwms
    loggy "$prolog"
    loggy "$prolog Docker container: $HYRAX_IMAGE STARTED."
    loggy "$prolog"
    loggy "$prolog END"

}

################################################################################
# enable_core_dump()
#
# Restarts the container with --privileged. Then it uses docker exec and
# bash to configure the running container to create core dump files in /cores
# on container's disk.
#
function enable_core_dump() {
    local prolog="enable_core_dump() -"
    loggy "$H2"
    loggy "$prolog BEGIN"
    run_privileged="--ulimit core=-1 --privileged --security-opt seccomp=unconfined "
    loggy "$prolog run_privileged: $run_privileged"
    stop_hyrax
    start_hyrax
    local naptime=10
    loggy "$prolog Waiting $naptime seconds for Hyrax to arrive..."
    sleep 10

    loggy ""
    loggy "$prolog Setting kernel.core.pattern in docker image:"
    loggy "$(docker exec -it "$HYRAX_NAME" bash -c "sysctl -w kernel.core_pattern=/cores/core-%e.%p.%h.%t; echo \"status: $?\"")"

    loggy ""
    loggy "$prolog docker ulimit: "
    loggy "$(docker exec -it "$HYRAX_NAME" bash -c "ulimit -a")"

    loggy ""
    loggy "$prolog host ulimit:"
    loggy "$(ulimit -a)"
    loggy ""

    loggy ""
    loggy "$prolog In docker container modify /etc/systemd/system.conf to all core dumps:"
    loggy "$(docker exec -it "$HYRAX_NAME" bash -c "echo \"DumpCore=yes\" >> /etc/systemd/system.conf; cat /etc/systemd/system.conf; ")"
    loggy ""

}

################################################################################
# check_for_aws_config()
#
function check_for_aws_config() {
    local prolog="check_for_aws_config() -"
    local status
    local msg

    msg="$(aws sts get-caller-identity 2>&1)"
    status=$?
    if test $status -ne 0; then
        loggy "$prolog ERROR! The AWS CLI is not configured unable to use AWS CLI."
        loggy "$prolog  AWS status: $status"
        loggy "$prolog AWS message: "$msg
    else
        loggy "$prolog AWS CLI is configured! user: "
        loggy "$msg"
    fi

    return $status
}

################################################################################
# install_linux_debug_packages()
#
# This is probably brittle - I think I got this list from gdb when
# I tried to look at a core dump with only the hyrax debuginfo RPMs
# installed. Given that the version numbers are "fully qualified" I
# suspect that when any of these packages gets updated the reference
# below will be out of date. For the core dump inspection the binary
# stuff has to match, or it's a debug train wreck.
#
function install_linux_debug_packages() {
    local prolog="install_linux_debug_packages() -"

    loggy "$prolog Installing various debug info..."
    docker exec -it hyrax bash -c "yum debuginfo-install -y \
        bzip2-libs-1.0.6-28.el8_10.x86_64 \
        glibc-2.28-251.el8_10.25.x86_64 \
        keyutils-libs-1.5.10-9.el8.x86_64 \
        krb5-libs-1.18.2-32.el8_10.x86_64 \
        libcom_err-1.45.6-6.el8_10.x86_64 \
        libcurl-minimal-7.61.1-34.el8_10.3.x86_64 \
        libgcc-8.5.0-28.el8_10.x86_64 \
        libicu-60.3-2.el8_1.x86_64 \
        libnghttp2-1.33.0-6.el8_10.1.x86_64 \
        libselinux-2.9-10.el8_10.x86_64 \
        libstdc++-8.5.0-28.el8_10.x86_64 \
        libtirpc-1.1.4-12.el8_10.x86_64 \
        libuuid-2.32.1-46.el8.x86_64 \
        libxml2-2.9.7-21.el8_10.3.x86_64 \
        libzstd-1.4.4-1.el8.x86_64 \
        openssl-libs-1.1.1k-14.el8_10.x86_64 \
        pcre2-10.32-3.el8_6.x86_64 \
        xz-libs-5.2.4-4.el8_6.x86_64 \
        zlib-1.2.11-26.el8.x86_64"
}

################################################################################
# enable_debugging()
# Installs all the poackages
# This will call enable_core_dump() which stops and then starts the server
# so the server should available for interrogation and configuration.
#
function enable_debugging() {
    local prolog="enable_debugging() -"
    local status
    loggy "$H1"
    loggy "$prolog BEGIN"
    loggy ""

    #-----------------------------------------------------------------
    # Make sure we have AWS CLI so we can get rpm files from our s3 bucket
    check_for_aws_config
    status=$?
    if test $status -ne 0; then
        loggy "$prolog ERROR! AWS CLI Not configured."
        # loggy  "$prolog status: $status"
        return $status
    fi
    loggy ""

    #-----------------------------------------------------------------
    # Enable core dumps
    enable_core_dump

    #-----------------------------------------------------------------
    # We grab the version response from the running server
    local version_response
    local version_url="http://localhost:8080/opendap/version"
    local version_file="./hyrax-version.xml"
    http_code=$(curl -o "%{http_code}" -o "$version_file" "$version_url")
    if test $http_code -ne 200; then
        loggy "ERROR! Failed to get Hyrax version response from: $version_url"
    else
        loggy "$prolog Hyrax Version Response:"
        loggy "$version_response"
    fi

    #-----------------------------------------------------------------
    # Using the version response document, get and install the
    # libdap4 debuginfo rpm
    local libdap4_version
    libdap4_version=$(echo "$version_response" | grep "library name=\"libdap\"" | awk '{ split($3,v,"="); gsub("\"","",v[2]); print v[2] ;}')
    echo "$prolog    libdap_version: $libdap4_version"

    local libdap_debug_rpm="libdap-debuginfo-$libdap4_version.el8.x86_64.rpm"
    loggy "$prolog libdap_debug_rpm: $libdap_debug_rpm"

    loggy "$prolog       Retrieving: $libdap_debug_rpm"
    aws s3 cp "$RPM_BUILD_REPO/$libdap_debug_rpm" "$bes_cores_dir"

    loggy "$prolog       Installing: $libdap_debug_rpm"
    docker exec -it hyrax bash -c "yum install -y /cores/$libdap_debug_rpm"

    #-----------------------------------------------------------------
    # Using the version response document, get and install the
    # BES debuginfo rpm
    local bes_version
    bes_version=$(echo "$version_response" | grep "library name=\"bes\"" | awk '{ split($3,v,"="); gsub("\"","",v[2]); print v[2]; }')
    echo "$prolog       bes_version: $bes_version"

    local bes_debug_rpm="bes-debuginfo-$bes_version.static.el8.x86_64.rpm"
    loggy "$prolog    bes_debug_rpm: $bes_debug_rpm"

    loggy "$prolog       Retrieving: $bes_debug_rpm"
    aws s3 cp "$RPM_BUILD_REPO/$bes_debug_rpm" "$bes_cores_dir"

    loggy "$prolog       Installing: $bes_debug_rpm"
    docker exec -it hyrax bash -c "yum install -y /cores/$bes_debug_rpm"

    #-----------------------------------------------------------------
    # Get the ancillary debugging symbols.
    install_linux_debug_packages

    loggy "$prolog END"

    return $status
}

################################################################################
#
# stop_hyrax()
#
function stop_hyrax() {
    local prolog="stop_hyrax() -"
    loggy "$prolog BEGIN"
    # Dump Hyrax container
    loggy "$prolog Forcibly removing the container: $HYRAX_NAME"
    loggy "$prolog Removed: $(docker rm -f "$HYRAX_NAME")"
    loggy "$prolog END"
}
################################################################################
#
# update_hyrax()
#
function update_hyrax() {
    local prolog="update_hyrax() -"
    loggy "$prolog BEGIN"
    docker pull "$HYRAX_IMAGE"
    stop_hyrax
    start_hyrax
    loggy "$prolog END"
}

################################################################################
#
# start_all()
#
function start_all() {
    local prolog="start_all() -"
    loggy "$prolog BEGIN"
    start_hyrax
    start_httpd
    loggy "$prolog END"
}

################################################################################
#
# stop_all()
#
function stop_all() {
    loggy "stop_all() - BEGIN"
    # Dump all containers
    docker rm -f $(docker ps -aq)
    # Dump the network.
    docker network rm "$HYRAX_NETWORK"
    loggy "stop_all() - END"
}

################################################################################
# clean_and_update()
#
#  - Stops the server
#  - Removes all local images
#  - Pulls $HYRAX_IMAGE
#  - Pulls $HTTPD_TIMAGE
#  - returns
#
function clean_and_update_all() {
    local prolog="clean_and_update_all() -"
    loggy "$prolog BEGIN"
    stop_all
    # Dump ALL the images
    docker rmi -f $(docker images -q)
    # Pull our stuff
    loggy "$prolog Pulling hyrax image: $HYRAX_IMAGE"
    docker pull "$HYRAX_IMAGE"
    loggy "$prolog Pulling httpd image: $HTTPD_IMAGE"
    docker pull "$HTTPD_IMAGE"
    loggy "$prolog END"
}

################################################################################
# Opens an interactive bash shell on the Hyrax container.
function dshell() {
    loggy "Logging into: $HYRAX_NAME"
    loggy ""
    docker exec -it "$HYRAX_NAME" /bin/bash
}

################################################################################
# Passes all the parameters to a bash shell in the HYRAX_NAME container.
function dsh() {
    loggy "Logging into: $HYRAX_NAME"
    loggy ""
    docker exec -it "$HYRAX_NAME" /bin/bash -c "$@"
}
function usage_all() {
    local usage_all
    usage_all=$(
        cat << EOF
NAME
  hyraxctl - hyrax control, all defined functions

FUNCTIONS:
    check_for_aws_config
        Checks and returns non zero status if aws cli is not configured.

    clean_and_update_all
        Stops and removes all containers and local images. Updates images. No restart.

    clean_hyrax_logs
        Removes all of the persistent log files for clean restart.

    config_ndp
        Config for Potter's situation

    config_ndp_local
        Config for Potter's situation

    config_rocky_8
    config_rocky_8_local

    config_test_opendap_org
        This is the thing then.

    config_tomcat_server

    delete_network
        Deletes docker network

    dsh
        New version of dshell that takes a bash script as a parameter

    dshell
        opens an interactive shell with the HYRAX_NAME docker container

    enable_core_dump
        Configures the container OS to create a stored core dump files.

    enable_debugging
        Enables runtime debugging by installing debug rpms and calling enable_core_dump

    get_platform
        Get the dpcker --platform switch if the host is an arm64

    get_recent_docker_image_tag_names
        Scans org/repo (opendap/hyrax) for the most recent tags.

    install_linux_debug_packages
        Just what it says, used by enabled_debugging

    loggy
        Simple output formatter

    make_network_as_needed
        If not present, makes a docker network for our httpd and hyrax deployments
        to utilize.

    run_cmd
        I don\'t know if we ever use this scriopt this way
        deprecated?

    save_hyrax_logs
        Saves the various hyrax logs to the crash_logs directory.

    set_file_permissions
        Starts hyrax, watches the docker logs, collects the uid:gid of the bes, and
        then massages the permissions on the various data mounts to match.

    show_config
        Show the current configuration state. Like a filtered pretty print env

    start_all
        Start httpd and hyrax, make_network_as_needed.

    start_debug
        Start hyrax with debugging enabled (rocky only)

    start_httpd
        Start the Apache Web Server, httpd.

    start_hyrax
        Start the hyrax container.

    stop_all
        Forcibly stops amd removes all of the containers.

    stop_httpd
        Stop the Apache Web Server, httpd.

    stop_hyrax
        Stop the hyrax container

    update_httpd
        Update the https docker image.

    update_hyrax
        Update the hyrax docker image.

    usage
        Print the basic usage message.

    usage_all
        Print this message
EOF
    )

    # Do the usage/warning things...
    loggy "$H1"
    loggy "$usage_all"
    if test "$USER" != "root";
    then
        loggy "$warning_doc"
    fi
    loggy "$H3"

}

################################################################################
#
# usage()
#  Written as a HERE document just to simplify the editing.
#
function usage() {

    # Slurp up a HERE document for the usage text output.
    local usage_doc
    usage_doc=$(
        cat << EOF
NAME
  hyraxctl - hyrax control script

SYNOPSIS
  hyraxctl cmd1 [cmd2 ... cmdN]

DESCRIPTION
  The hyraxctl utility implements a collection of commands that
  allow one to start, stop, and update Hyrax and Apache httpd docker
  images with all of the configuration and resources needed to
  operate test.opendap.org.
  The commands can be invoked by using one or more in the hyraxctl
  commandline line invocation:

      ./hyraxctl show_config

  Or, for two or more commands:

      ./hyraxctl config_test_opendap_org start_all

  Primary configuration values can be exported in the parent shell
  in order to inject different. Additionally, the file can be
  sourced:

      source ./hyraxctl
      . ./hyraxctl

  After which all of the commands will be directly available in the current
  shell:
      # . ./hyraxctl
      # show_config
      # start_all
      # stop_all

   The available commands follow:

            show_config - Sends the configuration state to stderr.
config_test_opendap_org - Configures the shell for deployment on
                          test.opendap.org (ubuntu Docker host)
       config_ndp_local - Configures the shell for dev deployment
                          on ndp\'s system.
   config_rocky_8_local - Configures the shell for aam AWS rocky8 system
                          and user rocky.

       enable_debugging - Launches an instance set up with debuginfo symbols
                          and core dumps enabled. AWS CLI must be conifgured!

              start_all - Makes the docker network, starts Hyrax,
                          starts httpd.
               stop_all - Stops and deletes all containers, deletes
                          docker network.
            start_hyrax - Starts Hyrax in the docker engine using
                          configuration values in the env.
             stop_hyrax - Deletes the running (or not) hyrax
                          container from the docker engine.
           update_hyrax - Pulls the latest HYRAX_IMAGE tag (currently:
                          $HYRAX_IMAGE), stops the Hyrax container,
                          and then starts the (possibly updated) image.
            start_httpd - Starts httpd in the docker engine using
                          configuration values in the env.
             stop_httpd - Deletes the running (or not) httpd
                          container from the docker engine.
           update_httpd - Pulls the latest HYRAX_IMAGE tag (currently:
                          $HTTPD_IMAGE), stops the Hyrax container,
                          and then starts the (possibly updated) image.
   clean_and_update_all - Stops and deletes all containers, deletes
                          docker network, deletes all images, pulls
                          Hyrax and httpd by tag

 make_network_as_needed - Creates docker network for Hyrax and
                          httpd, as needed.
         delete_network - Deletes the docker network for Hyrax
                          and httpd.
EOF
    )

    # Slurp up a HERE document for the warning text output.
    warning_doc=$(  cat << EOF
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
  WARNING: Your current user-id is: '$USER'
  WARNING: You must be the root user in order to use the functions in this script.
!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
EOF
    )
    # Do the usage/warning things...
    loggy "$H1"
    loggy "$usage_doc"
    if test "$USER" != "root"; then
        loggy "$warning_doc"
    fi
    loggy "$H3"
}

function run_cmd() {
    local cmd="$1"

    if test "$cmd" == "show_config"; then
        show_config
    elif test "$cmd" == "config_test_opendap_org"; then
        config_test_opendap_org
    elif test "$cmd" == "config_ndp_local"; then
        config_ndp_local
    elif test "$cmd" == "start_all"; then
        start_all
    elif test "$cmd" == "stop_all"; then
        stop_all
    elif test "$cmd" == "start_hyrax"; then
        start_hyrax
    elif test "$cmd" == "stop_hyrax"; then
        stop_hyrax
    elif test "$cmd" == "start_httpd"; then
        start_httpd
    elif test "$cmd" == "stop_httpd"; then
        stop_httpd
    elif test "$cmd" == "clean_and_update"; then
        clean_and_update
    elif test "$cmd" == "make_network_as_needed"; then
        make_network_as_needed
    elif test "$cmd" == "delete_network"; then
        delete_network
    else
        echo "ERROR Unrecognized command: $cmd" >&2
        usage
    fi
}

echo "ArgCount: $#" >&2
if test "$#" -gt 0; then
    for a_cmd in "$@"; do
        echo "Running command: $a_cmd" >&2
        run_cmd "$a_cmd"
    done
else
    config_test_opendap_org
    usage
fi

# libdap: 3.21.1-273
# bes: 3.21.1-653