From 3a7eb3b2677a5ba51536b67532ece4ac1c87ede8 Mon Sep 17 00:00:00 2001 From: mirsal Date: Wed, 2 Dec 2020 11:54:30 +0000 Subject: [PATCH 1/2] watch: Add an archive mode This commit adds an archive mode in which there is no initial sync and aws sync commands are run without the --delete flag, allowing for continuous archiving of local content to object storage. --- watch | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/watch b/watch index a9afcab..6de87ce 100755 --- a/watch +++ b/watch @@ -9,6 +9,9 @@ cat <<-EOF --force-restore restore even if local directory is not empty + --archive disable initial restore and do not delete + remote content that's missing locally + eg: $PROGNAME /data s3://bucket/dir EOF } @@ -18,7 +21,7 @@ function error_exit { exit 1 } -PARSED_OPTIONS=$(getopt -n "$0" -o f --long "force-restore" -- "$@") +PARSED_OPTIONS=$(getopt -n "$0" -o fa --long "force-restore,archive" -- "$@") if [ $? -ne 0 ]; then exit 1 fi @@ -30,6 +33,10 @@ while true; do FORCE_RESTORE="true" shift;; + -a|--archive) + ARCHIVE="true" + shift;; + --) shift break;; @@ -46,6 +53,12 @@ else AWS=aws fi +if [ ${ARCHIVE:false} == 'true' ]; then + DELETE="--delete" +else + DELETE= +fi + function restore { if [ "$(ls -A $LOCAL)" ]; then if [[ ${FORCE_RESTORE:false} == 'true' ]]; then @@ -61,7 +74,7 @@ function restore { function backup { echo "backup $LOCAL => $REMOTE" - if ! $AWS s3 sync "$LOCAL" "$REMOTE" --delete; then + if ! $AWS s3 sync "$LOCAL" "$REMOTE" $DELETE; then echo "backup failed" 1>&2 return 1 fi @@ -69,7 +82,7 @@ function backup { function final_backup { echo "backup $LOCAL => $REMOTE" - while ! $AWS s3 sync "$LOCAL" "$REMOTE" --delete; do + while ! $AWS s3 sync "$LOCAL" "$REMOTE" $DELETE; do echo "backup failed, will retry" 1>&2 sleep 1 done @@ -85,7 +98,9 @@ function idle { done } -restore +if [ ${ARCHIVE:false} == 'false' -o ${FORCE_RESTORE:false} == 'true' ]; then + restore +fi trap final_backup SIGHUP SIGINT SIGTERM trap "backup; idle" USR1 From 35d489bc5bc7a35ce000b183da8c42696b936fc7 Mon Sep 17 00:00:00 2001 From: mirsal Date: Wed, 2 Dec 2020 12:41:03 +0000 Subject: [PATCH 2/2] README.md: Document the archive option Add a paragraph describing how the archive mode works --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 6275e72..cb04bcd 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,13 @@ docker run -d --name my-data-container \ elementar/s3-volume --force-restore /data s3://mybucket/someprefix ``` +### Running in archive mode + +If the `--archive` option is set, the container will run without performing an +initial restore, even if the data directory is empty. Additionally, running in +archive mode disables the delete flag on aws s3 sync requests, so remote content +will not be deleted even if it is not present locally. + ### Using Compose and named volumes Most of the time, you will use this image to sync data for another container.