diff --git a/pillar/registry.sls b/pillar/registry.sls index 9736888e..b8e1d9f5 100644 --- a/pillar/registry.sls +++ b/pillar/registry.sls @@ -55,6 +55,7 @@ sync: /home/collect/scrapyd/dbs: /home/collect/scrapyd/eggs: /home/collect/scrapyd/jobs: + exclude: "*/requests.queue/*" /home/collect/scrapyd/logs: apache: diff --git a/salt/aws/files/sync-to-s3.sh b/salt/aws/files/sync-to-s3.sh index 96a35d5f..d3a4efc4 100644 --- a/salt/aws/files/sync-to-s3.sh +++ b/salt/aws/files/sync-to-s3.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Backup multiple directories and upload to AWS S3 +# Sync a single directory into S3. set -euo pipefail @@ -24,16 +24,42 @@ if [ ! -x "$AWS_CLI" ]; then exit 3 fi -if [ -z "$SYNC_DIRECTORIES" ]; then - echo "Error: SYNC_DIRECTORIES isn't set or is empty" +if [ -z "$S3_SYNC_BUCKET" ]; then + echo "Error: S3_SYNC_BUCKET isn't set or is empty" exit 4 fi -for DIRECTORY in "${SYNC_DIRECTORIES[@]}"; do - SAFENAME="${DIRECTORY/#\//}" - SAFENAME="${SAFENAME/%\//}" +AWS_ARGS=(--only-show-errors --delete) - set +e - $AWS_CLI s3 sync "$DIRECTORY" "s3://$S3_SYNC_BUCKET/$SAFENAME/" --only-show-errors --delete 2>&1 | grep -v "You did not provide the number of bytes specified by the Content-Length HTTP header" - set -e +# Parse script arguments +while [[ $# -gt 0 ]]; do + case $1 in + --exclude) + if [ -z "$2" ]; then + echo "Missing path." + exit 1 + fi + AWS_ARGS+=(--exclude "\"$2\"") + shift 2 + ;; + -*) + echo "Unknown option" + exit 1 + ;; + *) + DIRECTORY=$1 + SAFENAME=${DIRECTORY/#\//} + SAFENAME="${SAFENAME/%\//}" + shift + ;; + esac done + +if [ -e "$DIRECTORY" ]; then + echo "Error: DIRECTORY isn't set or doesn't exist." + exit 5 +fi + +set +e +$AWS_CLI s3 sync "${AWS_ARGS[@]}" "$DIRECTORY" "s3://$S3_SYNC_BUCKET/$SAFENAME/" +set -e diff --git a/salt/aws/sync.sls b/salt/aws/sync.sls index be5a4330..8869a57c 100644 --- a/salt/aws/sync.sls +++ b/salt/aws/sync.sls @@ -16,16 +16,10 @@ include: file.managed: - contents: | MAILTO=root - 15 03,15 * * * root /home/sysadmin-tools/bin/sync-to-s3.sh +{%- for directory, entry in pillar.sync.directories|items %} +{%- set minute = (loop.index0 * 5) % 60 %} + {{minute}} 03,15 * * * root /home/sysadmin-tools/bin/sync-to-s3.sh {{ directory }} + {%- for option, value in (entry or {}) | items %} --{{ option }} "{{ value }}"{% endfor %} +{%- endfor %} - require: - file: /home/sysadmin-tools/bin/sync-to-s3.sh - -set SYNC_DIRECTORIES setting: - file.keyvalue: - - name: /home/sysadmin-tools/aws-settings.local - - key: SYNC_DIRECTORIES - - value: '( "{{ pillar.sync.directories|join('" "') }}" )' - - append_if_not_found: True - - require: - - file: /home/sysadmin-tools/bin - - sls: aws