lcodeca · swangcs · Aug 3, 2020 · Aug 3, 2020 · Aug 10, 2020 · Aug 10, 2020
@@ -0,0 +1,176 @@
+#!/bin/bash
+
+# Example of docker usage for RLLIB + SUMO Utlis
+#
+# Author: Lara CODECA
+#
+# This program and the accompanying materials are made available under the
+# terms of the Eclipse Public License 2.0 which is available at
+# http://www.eclipse.org/legal/epl-2.0.
+
+set -e
+set -u
+
+IMAGE_NAME="tf-gpu-sumo-$(date +%Y-%m-%d)"
+IMAGE_FOLDER="docker-image-mac"
+GPU=true
+GPU_OPT="--gpus all"
+BUILD=false
+CACHE=false
+RUN=false
+SCREEN=false
+EXEC=false
+CONTAINER=""
+DEVEL_DIR=""
+LEARN_DIR=""
+USER_ID=""
+GROUP_ID=""
+COMMAND=""
+EXP=""
+DETACH=false
+SHM_SIZE="10g"
+
+function print_help {
+    echo "Parameters:"
+    echo "  IMAGE name \"$IMAGE_NAME\" [-n, --image-name]"
+    echo "  IMAGE folder \"$IMAGE_FOLDER\" [-f, --image-folder]"
+    echo "  GPU enabled ($GPU) [--no-gpu]"
+    echo "  BUILD: $BUILD [-b, --build], with CACHE: $CACHE [-c, --cache]"
+    echo "  RUN: $RUN [-r, --run], with SCREEN: $SCREEN [-s, --screen]"
+    echo "  EXEC: $EXEC [-e, --exec], CONTAINER: \"$CONTAINER\" (use docker ps for the id)"
+    echo "  COMMAND: \"$COMMAND\" [--cmd]"
+    echo "  EXP: \"$EXP\" [--exp]"
+    echo "  DETACH: ($DETACH) [--detach]"
+    echo "  DEVELOPMENT dir \"$DEVEL_DIR\" [-d, --devel]"
+    echo "  LEARNING dir \"$LEARN_DIR\" [-l, --learn]"
+    echo "  USER ID \"$USER_ID\" [-u, --uid]"
+    echo "  GROUP ID \"$GROUP_ID\" [-g, --gid]"
+    echo "  SHM_SIZE \"$SHM_SIZE\" [--shm-size]"
+}
+
+for arg in "$@"
+do
+    case $arg in ## -l=*|--lib=*) DIR="${i#*=}" is the way to retrieve the parameter
+        -n=*|--image-name=*)
+        IMAGE_NAME="${arg#*=}"
+        ;;
+        -f=*|--image-folder=*)
+        IMAGE_FOLDER="${arg#*=}"
+        ;;
+        --no-gpu)
+        GPU=false
+        GPU_OPT=""
+        ;;
+        --detach)
+        DETACH=true
+        ;;
+        -b|--build)
+        BUILD=true
+        ;;
+        -c|--cache) # it does nothing without BUILD=true
+        CACHE=true
+        ;;
+        -r|--run)
+        RUN=true
+        ;;
+        -s|--screen) # it does nothing without RUN=true
+        SCREEN=true
+        ;;
+        -e=*|--exec=*) # it works only with RUN=false
+        EXEC=true
+        CONTAINER="${arg#*=}"
+        ;;
+        --cmd=*)
+        COMMAND="${arg#*=}"
+        ;;
+        --exp=*)
+        EXP="${arg#*=}"
+        ;;
+        -d=*|--devel=*)
+        DEVEL_DIR="${arg#*=}"
+        ;;
+        -l=*|--learn=*)
+        LEARN_DIR="${arg#*=}"
+        ;;
+        -u=*|--uid=*)
+        USER_ID="${arg#*=}"
+        ;;
+        -g=*|--gid=*)
+        GROUP_ID="${arg#*=}"
+        ;;
+        --shm-size=*)
+        SHM_SIZE="${arg#*=}"
+        ;;
+        *)
+        # unknown option
+        echo "Unknown option \"$arg\""
+        print_help
+        exit
+        ;;
+    esac
+done
+
+print_help
+
+# Tensorflow original image
+# docker run -u $(id -u):$(id -g) --gpus all -it --rm tensorflow/tensorflow:latest-gpu-py3 bash
+
+## Building the docker image
+if [[ "$BUILD" = true ]]; then
+    if [[ "$CACHE" = true ]]; then
+        echo "Building the docker container using the cache, if present."
+        docker build \
+            --build-arg USER_ID="$USER_ID" \
+            --build-arg GROUP_ID="$GROUP_ID" \
+            -t "$IMAGE_NAME" "$IMAGE_FOLDER"
+    else
+        echo "Building the docker container ignoring the cache, even if present."
+        docker build \
+            --build-arg USER_ID="$USER_ID" \
+            --build-arg GROUP_ID="$GROUP_ID" \
+            --no-cache -t "$IMAGE_NAME" "$IMAGE_FOLDER"
+    fi
+fi
+
+if [[ "$RUN" = true ]]; then
+    # My docker build
+    MOUNT_DEVEL=""
+    if [[ $DEVEL_DIR ]]; then
+        MOUNT_DEVEL="--mount src=$DEVEL_DIR,target=/home/alice/devel,type=bind"
+    fi
+    MOUNT_LEARN=""
+    if [[ $LEARN_DIR ]]; then
+        MOUNT_LEARN="--mount src=$LEARN_DIR,target=/home/alice/learning,type=bind"
+    fi
+    CONT_NAME=""
+    if [[ $EXP ]]; then
+        CONT_NAME="--name $EXP"
+    fi
+    if [[ "$DETACH" = true ]]; then
+        DETACH="-d"
+    else
+        DETACH=""
+    fi
+    CURR_UID=$USER_ID
+    CURR_GID=$GROUP_ID
+    RUN_OPT="-u $CURR_UID:$CURR_GID --net=host --env DISPLAY=$DISPLAY \
+            --volume /tmp/.X11-unix:/tmp/.X11-unix \
+            --privileged $MOUNT_DEVEL $MOUNT_LEARN \
+            --shm-size $SHM_SIZE $GPU_OPT $CONT_NAME \
+            -it $DETACH --rm $IMAGE_NAME:latest"
+    echo "docker run $RUN_OPT $COMMAND"
+
+    ## Running docker
+    if [[ "$SCREEN" = true ]]; then
+        echo "Running the docker in a screen session."
+        screen -d -m \
+            docker run $RUN_OPT $COMMAND
+    else
+        docker run $RUN_OPT $COMMAND
+    fi
+else
+    if [[ "$EXEC" = true ]]; then
+        echo "Attaching to a running docker (see container id using 'docker ps')."
+        docker exec -it "$CONTAINER" /bin/bash
+    fi
+fi
@@ -0,0 +1,109 @@
+FROM tensorflow/tensorflow:latest-gpu-py3
+
+LABEL Author="Lara CODECA (lara.codeca@gmail.com)"
+LABEL Description="Dockerised SUMO-devel with tensorflow-gpu environment."
+
+ARG USER_ID
+ARG GROUP_ID
+
+# Install system dependencies.
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install \
+    cmake \
+    gdb \
+    git \
+    htop \
+    ipython3 \
+    libavcodec-dev \
+    libavformat-dev \
+    libavutil-dev \
+    libfox-1.6-0 \
+    libfox-1.6-dev \
+    libgdal-dev \
+    libopenmpi-dev \
+    libspatialindex-dev \
+    libswscale-dev \
+    libtool \
+    libxerces-c-dev \
+    nano \
+    psmisc \
+    python3.6-dev \
+    python3-tk \
+    python3-virtualenv \
+    rsync \
+    screen \
+    sudo \
+    swig \
+    tmux \
+    tree \
+    vim \
+    x11-apps \
+    zlib1g-dev && apt-get autoremove -y
+
+# Install Python 3 dependencies for SUMO and scripts
+RUN pip install --upgrade pip
+RUN python -m pip install \
+    aiohttp \
+    deepdiff \
+    dill \
+    folium \
+    gputil \
+    grpcio \
+    lxml \
+    lz4 \
+    matplotlib \
+    numpy \
+    opencv-python \
+    pandas \
+    psutil \
+    pyproj \
+    rtree \
+    setproctitle \
+    shapely \
+    tqdm
+
+# Install Python 3 dependencies for MARL
+RUN python -m pip install gym
+RUN python -m pip install ray
+RUN python -m pip install ray[debug]
+RUN python -m pip install ray[rllib]
+RUN python -m pip install ray[tune]
+
+# Working user
+RUN groupadd --gid ${GROUP_ID} alice && \
+    useradd -m -s /bin/bash -u ${USER_ID} -g ${GROUP_ID} alice && \
+    echo "alice:alice" | chpasswd && adduser alice sudo
+USER alice
+
+# Download and install SUMO Version v1_6_0
+WORKDIR /home/alice
+RUN git clone --depth 1 --branch v1_6_0 https://github.com/eclipse/sumo.git sumo
+RUN mkdir -p /home/alice/sumo/build/cmake-build-release
+WORKDIR /home/alice/sumo/build/cmake-build-release
+RUN cmake -D CHECK_OPTIONAL_LIBS=true -D CMAKE_BUILD_TYPE:STRING=Release /home/alice/sumo
+RUN make -j$(nproc)
+RUN mkdir -p /home/alice/sumo/build/cmake-build-debug
+WORKDIR /home/alice/sumo/build/cmake-build-debug
+RUN cmake -D CHECK_OPTIONAL_LIBS=true -D CMAKE_BUILD_TYPE:STRING=Debug /home/alice/sumo
+RUN make -j$(nproc)
+
+# Directory structure
+RUN mkdir -p /home/alice/devel
+RUN mkdir -p /home/alice/learning
+RUN mkdir -p /home/alice/libraries
+
+# Download & install RLLIB+SUMO Utils
+WORKDIR /home/alice/libraries
+RUN git clone --depth 1 https://github.com/lcodeca/rllibsumoutils.git rllibsumoutils
+WORKDIR /home/alice/libraries/rllibsumoutils
+USER root
+RUN python -m pip install -e .
+
+# Learning Environment
+USER alice
+WORKDIR /home/alice/learning
+COPY --chown=alice tf-gpu-test.py /home/alice/learning/tf-gpu-test.py
+COPY --chown=alice training.sh /home/alice/learning/training.sh
+
+USER alice
+WORKDIR /home/alice/learning
+CMD ["./training.sh"]
@@ -0,0 +1,3 @@
+import tensorflow as tf
+from tensorflow.python.client import device_lib
+print(device_lib.list_local_devices())
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# Example of learning script for RLLIB + SUMO Utlis
+#
+# Author: Lara CODECA
+#
+# This program and the accompanying materials are made available under the
+# terms of the Eclipse Public License 2.0 which is available at
+# http://www.eclipse.org/legal/epl-2.0.
+
+set -e
+
+# SUMO-dev environmental vars
+export SUMO_HOME="/home/alice/sumo"
+export SUMO_BIN="$SUMO_HOME/bin"
+export SUMO_TOOLS="$SUMO_HOME/tools"
+export PATH="$SUMO_BIN:$PATH"
+
+echo "Testing SUMO command:"
+sumo --version
+
+echo "Testing Tensorflow resources:"
+python tf-gpu-test.py
+
+echo "Testing RLLIB SUMO Utils example:"
+python /home/alice/libraries/rllibsumoutils/example/ppotrain.py
@@ -0,0 +1,67 @@
+# RLLIB SUMO Docker environment for MacOS
+
+### Author: Dr. Shen WANG (shen.wang@ucd.ie)
+
+## Step 1: Build your docker image
+
+Basic command to use:
+
+`bash docker-cmd-mac.sh --build --uid=502 --gid=222`
+
+- The default tag name of this image is `tf-gpu-sumo-{today_date}`. You can also add option such as `--image-name={new_name}` to specify your own name;
+- The default path of the dockerfile to be used is `docker-image-mac`. You can adapt to your setting by using option `--image-folder`;
+- The default image building process does not use cache. You can enable it by adding the option `--cache`.
+- The user id and group id are suggested as 502 and 222 respectively. The range of valid values on MacOS needs to be further studied.
+
+## Step 2: Run a container of the built image
+
+`sudo bash docker-cmd-mac.sh --image-name={your_image_name} --run --no-gpu  --uid=502 --gid=222`
+
+> :warning: **Using SUDO for docker is not a good practice**: Due to a problem of user privilege between host and container, `sudo` has to be used here to make it work. Feel free to let us know if you have any idea to solve this security issue.
+
+This will run `/home/alice/learning/training.sh` inside the docker container, which will then run `/home/alice/libraries/rllibsumoutils/example/ppotrain.py`.
+
+To facilitate your development, you can add options to synchronize the code on your local system with the docker container. The mappings are as follows:
+
+| Option      | Path (container) | Path (host)
+| ----------- | ----------- | ----------- |
+| [-d, --devel] | `/home/alice/devl/` | [your_path] |
+| [-l, --learn] | `/home/alice/learning/` | [your_path]        |
+
+
+
+
+## docker-cmd-mac.sh usage
+
+``` bash
+Default parameters:
+  IMAGE name "tf-gpu-sumo-{today}"      [-n, --image-name]
+  IMAGE folder "docker-image-mac"     [-f, --image-folder]
+  GPU enabled (true)                    [--no-gpu]
+  BUILD: false                          [-b, --build]
+        with CACHE: false               [-c, --cache]
+  RUN: false                            [-r, --run]
+        with SCREEN: false              [-s, --screen]
+  EXEC: false                           [-e, --exec]
+        CONTAINER: "" (use docker ps for the id)
+  COMMAND: ""                           [--cmd]
+  EXP: ""                               [--exp]
+  DETACH: (true)                        [--detach]
+  DEVELOPMENT dir ""                    [-d, --devel]
+  LEARNING dir ""                       [-l, --learn]
+  USER_ID ""                            [-u, --uid]
+  GROUP_ID ""                           [-g, --gid]
+```
+
+Example of use:
+
+- Build the image (with GPU enabled) and launch the example learning in the terminal:
+    `bash docker-cmd-mac.sh --build --cache --run`
+- Build the image with a specific name and launch the example learning in the terminal:
+    `bash docker-cmd-mac.sh --image-name=rllibsumoutils-docker --build --cache --run`
+- Run the already built image in a screen:
+    `bash docker-cmd-mac.sh --image-name=rllibsumoutils-docker --run --screen`
+- Attach to an already running docker container (using docker ps to know the id):
+    `bash docker-cmd-mac.sh --exec=9ed3cec06e`
+- The default docker entry point in the image runs the tests. It can be changed by using:
+    `bash docker-cmd-mac.sh --run --cmd="/bin/bash"`