From 613dba207afe56ad7f11392aa3f0153a2a69c57f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxime=20Gu=C3=A9riau?= Date: Fri, 7 Aug 2020 10:02:23 +0100 Subject: [PATCH 1/2] added documentation and scripts for windows --- docker-cmd-windows.sh | 179 ++++++++++++++++++++++++++++ docker-image-windows/Dockerfile | 124 +++++++++++++++++++ docker-image-windows/tf-gpu-test.py | 3 + docker-image-windows/training.sh | 26 ++++ docs/howto-windows.md | 28 +++++ 5 files changed, 360 insertions(+) create mode 100644 docker-cmd-windows.sh create mode 100644 docker-image-windows/Dockerfile create mode 100644 docker-image-windows/tf-gpu-test.py create mode 100644 docker-image-windows/training.sh create mode 100644 docs/howto-windows.md diff --git a/docker-cmd-windows.sh b/docker-cmd-windows.sh new file mode 100644 index 0000000..1a1d68b --- /dev/null +++ b/docker-cmd-windows.sh @@ -0,0 +1,179 @@ +#!/bin/bash + +# Example of docker usage for RLLIB + SUMO Utlis +# +# Author: Lara CODECA +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0. +# +# See https://github.com/lcodeca/rllibsumodocker/blob/master/docker-cmd.sh +# for the original. + +set -e + +IMAGE_NAME="tf-gpu-sumo-$(date +%Y-%m-%d)" +IMAGE_FOLDER="docker-image" +GPU=true +GPU_OPT="--gpus all" +OPTIRUN=false +OPTIRUN_OPT="" +BUILD=false +CACHE=false +RUN=false +SCREEN=false +EXEC=false +CONTAINER="" +DEVEL_DIR="$(pwd)/devel" +LEARN_DIR="$(pwd)/learning" +COMMAND="" +EXP="" +DETACH=false + +function print_help { + echo "Parameters:" + echo " IMAGE name \"$IMAGE_NAME\" [-n, --image-name]" + echo " IMAGE folder \"$IMAGE_FOLDER\" [-f, --image-folder]" + echo " GPU enabled ($GPU) [--no-gpu]" + echo " OPTIRUN disabled ($OPTIRUN) [--with-optirun]" + echo " BUILD: $BUILD [-b, --build], with CACHE: $CACHE [-c, --cache]" + echo " RUN: $RUN [-r, --run], with SCREEN: $SCREEN [-s, --screen]" + echo " EXEC: $EXEC [-e, --exec], CONTAINER: \"$CONTAINER\" (use docker ps for the id)" + echo " COMMAND: \"$COMMAND\" [--cmd]" + echo " EXP: \"$EXP\" [--exp]" + echo " DETACH: ($DETACH) [--detach]" + echo " DEVELOPMENT dir \"$DEVEL_DIR\" [-d, --devel]" + echo " LEARNING dir \"$LEARN_DIR\" [-l, --learn]" +} + +for arg in "$@" +do + case $arg in ## -l=*|--lib=*) DIR="${i#*=}" is the way to retrieve the parameter + -n=*|--image-name=*) + IMAGE_NAME="${arg#*=}" + ;; + -f=*|--image-folder=*) + IMAGE_FOLDER="${arg#*=}" + ;; + --no-gpu) + GPU=false + GPU_OPT="" + ;; + --with-optirun) + OPTIRUN=true + OPTIRUN_OPT="optirun" + ;; + --detach) + DETACH=true + ;; + -b|--build) + BUILD=true + ;; + -c|--cache) # it does nothing without BUILD=true + CACHE=true + ;; + -r|--run) + RUN=true + ;; + -s|--screen) # it does nothing without RUN=true + SCREEN=true + ;; + -e=*|--exec=*) # it works only with RUN=false + EXEC=true + CONTAINER="${arg#*=}" + ;; + --cmd=*) + COMMAND="${arg#*=}" + ;; + --exp=*) + EXP="${arg#*=}" + ;; + -d=*|--devel=*) + DEVEL_DIR="${arg#*=}" + ;; + -l=*|--learn=*) + LEARN_DIR="${arg#*=}" + ;; + *) + # unknown option + echo "Unknown option \"$arg\"" + print_help + exit + ;; + esac +done + +print_help + +# Tensorflow original image +# docker run -u $(id -u):$(id -g) --gpus all -it --rm tensorflow/tensorflow:latest-gpu-py3 bash + +## Building the docker image +if [[ "$BUILD" = true ]]; then + if [[ "$CACHE" = true ]]; then + echo "Building the docker container using the cache, if present." + $OPTIRUN_OPT docker build \ + --build-arg USER_ID=$(id -u ${USER}) \ + --build-arg GROUP_ID=$(id -g ${USER}) \ + -t "$IMAGE_NAME" "$IMAGE_FOLDER" + else + echo "Building the docker container ignoring the cache, even if present." + $OPTIRUN_OPT docker build \ + --build-arg USER_ID=$(id -u ${USER}) \ + --build-arg GROUP_ID=$(id -g ${USER}) \ + --no-cache -t "$IMAGE_NAME" "$IMAGE_FOLDER" + fi +fi + +if [[ "$RUN" = true ]]; then + # My docker build + MOUNT_DEVEL="" + if [[ $DEVEL_DIR ]]; then + MOUNT_DEVEL="--mount src=$DEVEL_DIR,target=/home/alice/devel,type=bind" + #MOUNT_DEVEL="-v $DEVEL_DIR:/home/alice/devel" + fi + MOUNT_LEARN="" + if [[ $LEARN_DIR ]]; then + MOUNT_LEARN="--mount src=$LEARN_DIR,target=/home/alice/learning,type=bind" + #MOUNT_DEVEL="-v $LEARN_DIR:/home/alice/learning" + fi + CONT_NAME="" + if [[ $EXP ]]; then + CONT_NAME="--name $EXP" + fi + if [[ "$DETACH" = true ]]; then + DETACH="-d" + else + DETACH="" + fi + CURR_UID=$(id -u) + echo $(id -u) + CURR_GID=$(id -g) + echo $(id -g) + RUN_OPT="-u $CURR_UID:$CURR_GID --net=host --env DISPLAY=$DISPLAY \ + --volume $XAUTHORITY:/home/alice/.Xauthority \ + --volume /tmp/.X11-unix:/tmp/.X11-unix \ + --privileged $MOUNT_DEVEL $MOUNT_LEARN \ + --shm-size 256m $GPU_OPT $CONT_NAME \ + -it $DETACH --rm $IMAGE_NAME:latest" + echo "$OPTIRUN_OPT docker run $RUN_OPT $COMMAND" + + ## Running docker + if [[ "$SCREEN" = true ]]; then + echo "Running the docker in a screen session." + screen -d -m \ + $OPTIRUN_OPT docker run $RUN_OPT $COMMAND + else + echo "Running the docker in this teminal." + echo $RUN_OPT + echo $COMMAND + echo "----------------------" + winpty docker run $RUN_OPT $COMMAND + fi +else + if [[ "$EXEC" = true ]]; then + echo "Attaching to a running docker (see container id using 'docker ps')." + $OPTIRUN_OPT docker exec -it "$CONTAINER" /bin/bash + fi +fi \ No newline at end of file diff --git a/docker-image-windows/Dockerfile b/docker-image-windows/Dockerfile new file mode 100644 index 0000000..409b8a3 --- /dev/null +++ b/docker-image-windows/Dockerfile @@ -0,0 +1,124 @@ +FROM tensorflow/tensorflow:latest-gpu-py3 + +LABEL Author="Lara CODECA (lara.codeca@gmail.com)" +LABEL Description="Dockerised SUMO-devel with tensorflow-gpu environment, RLLIBSUMOUtils and PyPML." + +ARG USER_ID +ARG GROUP_ID + +# Install system dependencies. +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y install \ + cmake \ + gdb \ + git \ + htop \ + libavcodec-dev \ + libavformat-dev \ + libavutil-dev \ + libfox-1.6-0 \ + libfox-1.6-dev \ + libgdal-dev \ + libopenmpi-dev \ + libspatialindex-dev \ + libswscale-dev \ + libtool \ + libxerces-c-dev \ + nano \ + psmisc \ + python3.6-dev \ + python3-tk \ + python3-virtualenv \ + rsync \ + screen \ + sudo \ + swig \ + tmux \ + tree \ + vim \ + x11-apps \ + zlib1g-dev + +# Install Python 3 dependencies for SUMO and scripts +RUN pip install --upgrade pip +RUN python -m pip install \ + aiohttp \ + dill \ + folium \ + gputil \ + grpcio \ + lxml \ + lz4 \ + matplotlib \ + numpy \ + opencv-python \ + pandas \ + psutil \ + pyproj \ + rtree \ + setproctitle \ + shapely \ + tqdm + +# Install Python 3 dependencies for MARL +RUN python -m pip install gym +RUN python -m pip install ray +RUN python -m pip install ray[debug] +RUN python -m pip install ray[rllib] +RUN python -m pip install ray[tune] + +# Working user +#RUN groupadd --gid ${GROUP_ID} alice && \ +# useradd -m -s /bin/bash -u ${USER_ID} -g ${GROUP_ID} alice && \ +# echo "alice:alice" | chpasswd && adduser alice sudo +#USER alice + +# Working user +RUN groupadd --gid 197121 alice && \ + useradd -m -s /bin/bash -u 197108 -g 197121 alice && \ + echo "alice:alice" | chpasswd && adduser alice sudo +USER alice + +# Download and install SUMO +WORKDIR /home/alice +RUN git clone --depth 1 https://github.com/eclipse/sumo.git sumo +RUN mkdir -p /home/alice/sumo/build/cmake-build-release +WORKDIR /home/alice/sumo/build/cmake-build-release +RUN cmake -D CHECK_OPTIONAL_LIBS=true -D CMAKE_BUILD_TYPE:STRING=Release /home/alice/sumo +RUN make -j$(nproc) +RUN mkdir -p /home/alice/sumo/build/cmake-build-debug +WORKDIR /home/alice/sumo/build/cmake-build-debug +RUN cmake -D CHECK_OPTIONAL_LIBS=true -D CMAKE_BUILD_TYPE:STRING=Debug /home/alice/sumo +RUN make -j$(nproc) + +# Directory structure +USER alice +RUN mkdir -p /home/alice/devel +RUN mkdir -p /home/alice/learning +RUN mkdir -p /home/alice/libraries +RUN mkdir -p /home/alice/testing + +# Download & install RLLIB+SUMO Utils +USER alice +WORKDIR /home/alice/libraries +RUN git clone --depth 1 https://github.com/lcodeca/rllibsumoutils.git rllibsumoutils +WORKDIR /home/alice/libraries/rllibsumoutils +USER root +RUN python -m pip install -e . + +CMD ["export SUMO_HOME=/home/alice/sumo"] +# Download & install PyPML Utils +USER alice +WORKDIR /home/alice/libraries +RUN git clone --depth 1 https://github.com/lcodeca/PyPML pypml +WORKDIR /home/alice/libraries/pypml +USER root +RUN python -m pip install -e . + +# Testing Environment +USER alice +WORKDIR /home/alice/testing +COPY --chown=alice tf-gpu-test.py /home/alice/testing/tf-gpu-test.py + +USER alice +WORKDIR /home/alice/learning +CMD ["./training.sh"] diff --git a/docker-image-windows/tf-gpu-test.py b/docker-image-windows/tf-gpu-test.py new file mode 100644 index 0000000..0197f80 --- /dev/null +++ b/docker-image-windows/tf-gpu-test.py @@ -0,0 +1,3 @@ +import tensorflow as tf +from tensorflow.python.client import device_lib +print(device_lib.list_local_devices()) diff --git a/docker-image-windows/training.sh b/docker-image-windows/training.sh new file mode 100644 index 0000000..dcc4d24 --- /dev/null +++ b/docker-image-windows/training.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Example of learning script for RLLIB + SUMO Utlis +# +# Author: Lara CODECA +# +# This program and the accompanying materials are made available under the +# terms of the Eclipse Public License 2.0 which is available at +# http://www.eclipse.org/legal/epl-2.0. + +set -e + +# SUMO-dev environmental vars +export SUMO_HOME="/home/alice/sumo" +export SUMO_BIN="$SUMO_HOME/bin" +export SUMO_TOOLS="$SUMO_HOME/tools" +export PATH="$SUMO_BIN:$PATH" + +echo "Testing SUMO command:" +sumo --version + +echo "Testing Tensorflow resources:" +python tf-gpu-test.py + +echo "Testing RLLIB SUMO Utils example:" +python /home/alice/libraries/rllibsumoutils/example/ppotrain.py diff --git a/docs/howto-windows.md b/docs/howto-windows.md new file mode 100644 index 0000000..1a253c2 --- /dev/null +++ b/docs/howto-windows.md @@ -0,0 +1,28 @@ +# RLLIB SUMO Docker environment for Windows + + +Step by step configuration and use: + +1. install docker for windows + +(https://hub.docker.com/editions/community/docker-ce-desktop-windows/) + +2. install VcXsrv Windows X Server (https://sourceforge.net/projects/vcxsrv/files/latest/download) + +(follow steps from https://dev.to/darksmile92/run-gui-app-in-linux-docker-container-on-windows-host-4kde) + +3. build the docker image using the docker cmd bash script + +`./docker-cmd.sh -n=marl-parking-devel -b -c -r` + +4. use the following command line: to set up VcXsrv: + +`set-variable -name DISPLAY -value 192.168.xx.xx:0.0` + +(/!\ replace "192.168.xx.xx" with your ip you can obtain using ipconfig command line) + +5. use the following command line to run your docker image: + +(of course replace "GroupID, UserID and FolderPath" accordingly + +`docker run -u GroupID:UserID --net=host --privileged -v /c/FolderPath/MARL-Parking/rllibsumodocker/learning:/home/alice/learning -v /c/FolderPath/MARL-Parking/rllibsumodocker/devel:/home/alice/devel -it --rm -e DISPLAY=$DISPLAY marl-parking-devel:latest /bin/bash` \ No newline at end of file From 812ff9f21f8ec4ce28286608c353b47d62d76828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maxime=20Gu=C3=A9riau?= Date: Fri, 7 Aug 2020 10:08:42 +0100 Subject: [PATCH 2/2] Cleaned howto-windows.md --- docs/howto-windows.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/howto-windows.md b/docs/howto-windows.md index 1a253c2..b30135f 100644 --- a/docs/howto-windows.md +++ b/docs/howto-windows.md @@ -13,7 +13,7 @@ Step by step configuration and use: 3. build the docker image using the docker cmd bash script -`./docker-cmd.sh -n=marl-parking-devel -b -c -r` +`./docker-cmd-windows.sh -n=marl-parking-devel -b -c -r` 4. use the following command line: to set up VcXsrv: @@ -25,4 +25,4 @@ Step by step configuration and use: (of course replace "GroupID, UserID and FolderPath" accordingly -`docker run -u GroupID:UserID --net=host --privileged -v /c/FolderPath/MARL-Parking/rllibsumodocker/learning:/home/alice/learning -v /c/FolderPath/MARL-Parking/rllibsumodocker/devel:/home/alice/devel -it --rm -e DISPLAY=$DISPLAY marl-parking-devel:latest /bin/bash` \ No newline at end of file +`docker run -u GroupID:UserID --net=host --privileged -v /c/FolderPath/rllibsumodocker/learning:/home/alice/learning -v /c/FolderPath/rllibsumodocker/devel:/home/alice/devel -it --rm -e DISPLAY=$DISPLAY marl-parking-devel:latest /bin/bash` \ No newline at end of file