diff --git a/docker-cmd-mac.sh b/docker-cmd-mac.sh new file mode 100755 index 0000000..985b5b5 --- /dev/null +++ b/docker-cmd-mac.sh @@ -0,0 +1,176 @@ +#!/bin/bash + +# Example of docker usage for RLLIB + SUMO Utlis +# +# Author: Lara CODECA +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0. + +set -e +set -u + +IMAGE_NAME="tf-gpu-sumo-$(date +%Y-%m-%d)" +IMAGE_FOLDER="docker-image-mac" +GPU=true +GPU_OPT="--gpus all" +BUILD=false +CACHE=false +RUN=false +SCREEN=false +EXEC=false +CONTAINER="" +DEVEL_DIR="" +LEARN_DIR="" +USER_ID="" +GROUP_ID="" +COMMAND="" +EXP="" +DETACH=false +SHM_SIZE="10g" + +function print_help { + echo "Parameters:" + echo " IMAGE name \"$IMAGE_NAME\" [-n, --image-name]" + echo " IMAGE folder \"$IMAGE_FOLDER\" [-f, --image-folder]" + echo " GPU enabled ($GPU) [--no-gpu]" + echo " BUILD: $BUILD [-b, --build], with CACHE: $CACHE [-c, --cache]" + echo " RUN: $RUN [-r, --run], with SCREEN: $SCREEN [-s, --screen]" + echo " EXEC: $EXEC [-e, --exec], CONTAINER: \"$CONTAINER\" (use docker ps for the id)" + echo " COMMAND: \"$COMMAND\" [--cmd]" + echo " EXP: \"$EXP\" [--exp]" + echo " DETACH: ($DETACH) [--detach]" + echo " DEVELOPMENT dir \"$DEVEL_DIR\" [-d, --devel]" + echo " LEARNING dir \"$LEARN_DIR\" [-l, --learn]" + echo " USER ID \"$USER_ID\" [-u, --uid]" + echo " GROUP ID \"$GROUP_ID\" [-g, --gid]" + echo " SHM_SIZE \"$SHM_SIZE\" [--shm-size]" +} + +for arg in "$@" +do + case $arg in ## -l=*|--lib=*) DIR="${i#*=}" is the way to retrieve the parameter + -n=*|--image-name=*) + IMAGE_NAME="${arg#*=}" + ;; + -f=*|--image-folder=*) + IMAGE_FOLDER="${arg#*=}" + ;; + --no-gpu) + GPU=false + GPU_OPT="" + ;; + --detach) + DETACH=true + ;; + -b|--build) + BUILD=true + ;; + -c|--cache) # it does nothing without BUILD=true + CACHE=true + ;; + -r|--run) + RUN=true + ;; + -s|--screen) # it does nothing without RUN=true + SCREEN=true + ;; + -e=*|--exec=*) # it works only with RUN=false + EXEC=true + CONTAINER="${arg#*=}" + ;; + --cmd=*) + COMMAND="${arg#*=}" + ;; + --exp=*) + EXP="${arg#*=}" + ;; + -d=*|--devel=*) + DEVEL_DIR="${arg#*=}" + ;; + -l=*|--learn=*) + LEARN_DIR="${arg#*=}" + ;; + -u=*|--uid=*) + USER_ID="${arg#*=}" + ;; + -g=*|--gid=*) + GROUP_ID="${arg#*=}" + ;; + --shm-size=*) + SHM_SIZE="${arg#*=}" + ;; + *) + # unknown option + echo "Unknown option \"$arg\"" + print_help + exit + ;; + esac +done + +print_help + +# Tensorflow original image +# docker run -u $(id -u):$(id -g) --gpus all -it --rm tensorflow/tensorflow:latest-gpu-py3 bash + +## Building the docker image +if [[ "$BUILD" = true ]]; then + if [[ "$CACHE" = true ]]; then + echo "Building the docker container using the cache, if present." + docker build \ + --build-arg USER_ID="$USER_ID" \ + --build-arg GROUP_ID="$GROUP_ID" \ + -t "$IMAGE_NAME" "$IMAGE_FOLDER" + else + echo "Building the docker container ignoring the cache, even if present." + docker build \ + --build-arg USER_ID="$USER_ID" \ + --build-arg GROUP_ID="$GROUP_ID" \ + --no-cache -t "$IMAGE_NAME" "$IMAGE_FOLDER" + fi +fi + +if [[ "$RUN" = true ]]; then + # My docker build + MOUNT_DEVEL="" + if [[ $DEVEL_DIR ]]; then + MOUNT_DEVEL="--mount src=$DEVEL_DIR,target=/home/alice/devel,type=bind" + fi + MOUNT_LEARN="" + if [[ $LEARN_DIR ]]; then + MOUNT_LEARN="--mount src=$LEARN_DIR,target=/home/alice/learning,type=bind" + fi + CONT_NAME="" + if [[ $EXP ]]; then + CONT_NAME="--name $EXP" + fi + if [[ "$DETACH" = true ]]; then + DETACH="-d" + else + DETACH="" + fi + CURR_UID=$USER_ID + CURR_GID=$GROUP_ID + RUN_OPT="-u $CURR_UID:$CURR_GID --net=host --env DISPLAY=$DISPLAY \ + --volume /tmp/.X11-unix:/tmp/.X11-unix \ + --privileged $MOUNT_DEVEL $MOUNT_LEARN \ + --shm-size $SHM_SIZE $GPU_OPT $CONT_NAME \ + -it $DETACH --rm $IMAGE_NAME:latest" + echo "docker run $RUN_OPT $COMMAND" + + ## Running docker + if [[ "$SCREEN" = true ]]; then + echo "Running the docker in a screen session." + screen -d -m \ + docker run $RUN_OPT $COMMAND + else + docker run $RUN_OPT $COMMAND + fi +else + if [[ "$EXEC" = true ]]; then + echo "Attaching to a running docker (see container id using 'docker ps')." + docker exec -it "$CONTAINER" /bin/bash + fi +fi diff --git a/docker-image-mac/Dockerfile b/docker-image-mac/Dockerfile new file mode 100644 index 0000000..f17736a --- /dev/null +++ b/docker-image-mac/Dockerfile @@ -0,0 +1,109 @@ +FROM tensorflow/tensorflow:latest-gpu-py3 + +LABEL Author="Lara CODECA (lara.codeca@gmail.com)" +LABEL Description="Dockerised SUMO-devel with tensorflow-gpu environment." + +ARG USER_ID +ARG GROUP_ID + +# Install system dependencies. +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install \ + cmake \ + gdb \ + git \ + htop \ + ipython3 \ + libavcodec-dev \ + libavformat-dev \ + libavutil-dev \ + libfox-1.6-0 \ + libfox-1.6-dev \ + libgdal-dev \ + libopenmpi-dev \ + libspatialindex-dev \ + libswscale-dev \ + libtool \ + libxerces-c-dev \ + nano \ + psmisc \ + python3.6-dev \ + python3-tk \ + python3-virtualenv \ + rsync \ + screen \ + sudo \ + swig \ + tmux \ + tree \ + vim \ + x11-apps \ + zlib1g-dev && apt-get autoremove -y + +# Install Python 3 dependencies for SUMO and scripts +RUN pip install --upgrade pip +RUN python -m pip install \ + aiohttp \ + deepdiff \ + dill \ + folium \ + gputil \ + grpcio \ + lxml \ + lz4 \ + matplotlib \ + numpy \ + opencv-python \ + pandas \ + psutil \ + pyproj \ + rtree \ + setproctitle \ + shapely \ + tqdm + +# Install Python 3 dependencies for MARL +RUN python -m pip install gym +RUN python -m pip install ray +RUN python -m pip install ray[debug] +RUN python -m pip install ray[rllib] +RUN python -m pip install ray[tune] + +# Working user +RUN groupadd --gid ${GROUP_ID} alice && \ + useradd -m -s /bin/bash -u ${USER_ID} -g ${GROUP_ID} alice && \ + echo "alice:alice" | chpasswd && adduser alice sudo +USER alice + +# Download and install SUMO Version v1_6_0 +WORKDIR /home/alice +RUN git clone --depth 1 --branch v1_6_0 https://github.com/eclipse/sumo.git sumo +RUN mkdir -p /home/alice/sumo/build/cmake-build-release +WORKDIR /home/alice/sumo/build/cmake-build-release +RUN cmake -D CHECK_OPTIONAL_LIBS=true -D CMAKE_BUILD_TYPE:STRING=Release /home/alice/sumo +RUN make -j$(nproc) +RUN mkdir -p /home/alice/sumo/build/cmake-build-debug +WORKDIR /home/alice/sumo/build/cmake-build-debug +RUN cmake -D CHECK_OPTIONAL_LIBS=true -D CMAKE_BUILD_TYPE:STRING=Debug /home/alice/sumo +RUN make -j$(nproc) + +# Directory structure +RUN mkdir -p /home/alice/devel +RUN mkdir -p /home/alice/learning +RUN mkdir -p /home/alice/libraries + +# Download & install RLLIB+SUMO Utils +WORKDIR /home/alice/libraries +RUN git clone --depth 1 https://github.com/lcodeca/rllibsumoutils.git rllibsumoutils +WORKDIR /home/alice/libraries/rllibsumoutils +USER root +RUN python -m pip install -e . + +# Learning Environment +USER alice +WORKDIR /home/alice/learning +COPY --chown=alice tf-gpu-test.py /home/alice/learning/tf-gpu-test.py +COPY --chown=alice training.sh /home/alice/learning/training.sh + +USER alice +WORKDIR /home/alice/learning +CMD ["./training.sh"] diff --git a/docker-image-mac/tf-gpu-test.py b/docker-image-mac/tf-gpu-test.py new file mode 100755 index 0000000..0197f80 --- /dev/null +++ b/docker-image-mac/tf-gpu-test.py @@ -0,0 +1,3 @@ +import tensorflow as tf +from tensorflow.python.client import device_lib +print(device_lib.list_local_devices()) diff --git a/docker-image-mac/training.sh b/docker-image-mac/training.sh new file mode 100755 index 0000000..dcc4d24 --- /dev/null +++ b/docker-image-mac/training.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Example of learning script for RLLIB + SUMO Utlis +# +# Author: Lara CODECA +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0. + +set -e + +# SUMO-dev environmental vars +export SUMO_HOME="/home/alice/sumo" +export SUMO_BIN="$SUMO_HOME/bin" +export SUMO_TOOLS="$SUMO_HOME/tools" +export PATH="$SUMO_BIN:$PATH" + +echo "Testing SUMO command:" +sumo --version + +echo "Testing Tensorflow resources:" +python tf-gpu-test.py + +echo "Testing RLLIB SUMO Utils example:" +python /home/alice/libraries/rllibsumoutils/example/ppotrain.py diff --git a/docs/howto-mac.md b/docs/howto-mac.md new file mode 100644 index 0000000..396a58a --- /dev/null +++ b/docs/howto-mac.md @@ -0,0 +1,67 @@ +# RLLIB SUMO Docker environment for MacOS + +### Author: Dr. Shen WANG (shen.wang@ucd.ie) + +## Step 1: Build your docker image + +Basic command to use: + +`bash docker-cmd-mac.sh --build --uid=502 --gid=222` + +- The default tag name of this image is `tf-gpu-sumo-{today_date}`. You can also add option such as `--image-name={new_name}` to specify your own name; +- The default path of the dockerfile to be used is `docker-image-mac`. You can adapt to your setting by using option `--image-folder`; +- The default image building process does not use cache. You can enable it by adding the option `--cache`. +- The user id and group id are suggested as 502 and 222 respectively. The range of valid values on MacOS needs to be further studied. + +## Step 2: Run a container of the built image + +`sudo bash docker-cmd-mac.sh --image-name={your_image_name} --run --no-gpu --uid=502 --gid=222` + +> :warning: **Using SUDO for docker is not a good practice**: Due to a problem of user privilege between host and container, `sudo` has to be used here to make it work. Feel free to let us know if you have any idea to solve this security issue. + +This will run `/home/alice/learning/training.sh` inside the docker container, which will then run `/home/alice/libraries/rllibsumoutils/example/ppotrain.py`. + +To facilitate your development, you can add options to synchronize the code on your local system with the docker container. The mappings are as follows: + +| Option | Path (container) | Path (host) +| ----------- | ----------- | ----------- | +| [-d, --devel] | `/home/alice/devl/` | [your_path] | +| [-l, --learn] | `/home/alice/learning/` | [your_path] | + + + + +## docker-cmd-mac.sh usage + +``` bash +Default parameters: + IMAGE name "tf-gpu-sumo-{today}" [-n, --image-name] + IMAGE folder "docker-image-mac" [-f, --image-folder] + GPU enabled (true) [--no-gpu] + BUILD: false [-b, --build] + with CACHE: false [-c, --cache] + RUN: false [-r, --run] + with SCREEN: false [-s, --screen] + EXEC: false [-e, --exec] + CONTAINER: "" (use docker ps for the id) + COMMAND: "" [--cmd] + EXP: "" [--exp] + DETACH: (true) [--detach] + DEVELOPMENT dir "" [-d, --devel] + LEARNING dir "" [-l, --learn] + USER_ID "" [-u, --uid] + GROUP_ID "" [-g, --gid] +``` + +Example of use: + +- Build the image (with GPU enabled) and launch the example learning in the terminal: + `bash docker-cmd-mac.sh --build --cache --run` +- Build the image with a specific name and launch the example learning in the terminal: + `bash docker-cmd-mac.sh --image-name=rllibsumoutils-docker --build --cache --run` +- Run the already built image in a screen: + `bash docker-cmd-mac.sh --image-name=rllibsumoutils-docker --run --screen` +- Attach to an already running docker container (using docker ps to know the id): + `bash docker-cmd-mac.sh --exec=9ed3cec06e` +- The default docker entry point in the image runs the tests. It can be changed by using: + `bash docker-cmd-mac.sh --run --cmd="/bin/bash"`