#58 Multiple backup jobs support.

README.md

@@ -1,5 +1,6 @@
 # Blacklabelops Volumerize
 
+[![Circle CI](https://circleci.com/gh/blacklabelops/volumerize.svg?style=shield)](https://circleci.com/gh/blacklabelops/volumerize)
 [![Open Issues](https://img.shields.io/github/issues/blacklabelops/volumerize.svg)](https://github.com/blacklabelops/volumerize/issues) [![Stars on GitHub](https://img.shields.io/github/stars/blacklabelops/volumerize.svg)](https://github.com/blacklabelops/volumerize/stargazers)
 [![Docker Stars](https://img.shields.io/docker/stars/blacklabelops/volumerize.svg)](https://hub.docker.com/r/blacklabelops/volumerize/) [![Docker Pulls](https://img.shields.io/docker/pulls/blacklabelops/volumerize.svg)](https://hub.docker.com/r/blacklabelops/volumerize/)
 
@@ -475,6 +476,53 @@ $ docker exec volumerize backup --dry-run
 
 Check out the project at Github.
 
+# Multiple Backups
+
+You can specify multiple backup jobs with one container with enumerated environment variables. Each environment variable must be followed by a number starting with 1. Example `VOLUMERIZE_SOURCE1`, `VOLUMERIZE_SOURCE2` or `VOLUMERIZE_SOURCE3`.
+
+The following environment variables can be enumerated:
+
+* VOLUMERIZE_SOURCE
+* VOLUMERIZE_TARGET
+* VOLUMERIZE_CACHE
+* VOLUMERIZE_INCLUDE
+
+When using multiple backup jobs you will have to specify a cache directory for each backup. The minimum required environment variables for each job is:
+
+* VOLUMERIZE_SOURCE
+* VOLUMERIZE_TARGET
+* VOLUMERIZE_CACHE
+
+Also the included helper scripts will change their behavior when you use enumerated environment variables. By default each script will run on all backup jobs.
+
+Example: Executing the script `backup` will backup all jobs.
+
+The first parameter of each script can be a job number, e.g. `1`, `2` or `3`.
+
+Example: Executing the script `backup 1` will only trigger backup on job 1.
+
+Full example for multiple job specifications:
+
+~~~~
+$ docker run -d \
+    --name volumerize \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v jenkins_volume:/source:ro \
+    -v jenkins_volume2:/source2:ro \
+    -v backup_volume:/backup \
+    -v backup_volume2:/backup2 \
+    -v cache_volume:/volumerize-cache \
+    -v cache_volume2:/volumerize-cache2 \
+    -e "VOLUMERIZE_CONTAINERS=jenkins jenkins2" \
+    -e "VOLUMERIZE_SOURCE1=/source" \
+    -e "VOLUMERIZE_TARGET1=file:///backup" \
+    -e "VOLUMERIZE_CACHE1=/volumerize-cache" \
+    -e "VOLUMERIZE_SOURCE2=/source2" \
+    -e "VOLUMERIZE_TARGET2=file:///backup2" \
+    -e "VOLUMERIZE_CACHE2=/volumerize-cache2" \
+    blacklabelops/volumerize
+~~~~
+
 ## Build the Image
 
 ~~~~

buildscripts/release.sh

@@ -3,5 +3,5 @@
 #------------------
 # CONTAINER VARIABLES
 #------------------
-export IMAGE_VERSION=1.4.2
+export IMAGE_VERSION=1.5.0
 export BUILD_BRANCH=$(git branch | grep -e "^*" | cut -d' ' -f 2)

imagescripts/base.sh

@@ -43,5 +43,96 @@ function resolveIncludes() {
   done
 }
 
+JOB_COUNT=
+
+function discoverJobs() {
+  local x
+  for (( x=1; ; x++ ))
+  do
+    JOB_VARIABLE="VOLUMERIZE_SOURCE${x}"
+    if [ ! -n "${!JOB_VARIABLE}" ]; then
+      break
+    else
+      JOB_COUNT=$x
+    fi
+  done
+}
+
+DUPLICITY_JOB_COMMAND=
+DUPLICITY_JOB_OPTIONS=
+VOLUMERIZE_JOB_SOURCE=
+VOLUMERIZE_JOB_TARGET=
+VOLUMERIZE_JOB_INCUDES=
+
+function prepareJobCommand() {
+  local jobNumber=$1
+  DUPLICITY_JOB_COMMAND=$DUPLICITY_COMMAND
+  DUPLICITY_JOB_OPTIONS="--allow-source-mismatch"
+  local CACHE_VARIABLE="VOLUMERIZE_CACHE${jobNumber}"
+  if [ -n "${!CACHE_VARIABLE}" ]; then
+    DUPLICITY_JOB_OPTIONS=$DUPLICITY_JOB_OPTIONS" --archive-dir=${!CACHE_VARIABLE}"
+  else
+    DUPLICITY_JOB_OPTIONS=$DUPLICITY_JOB_OPTIONS" --archive-dir=${VOLUMERIZE_CACHE}"
+  fi
+  if [ -n "${VOLUMERIZE_DUPLICITY_OPTIONS}" ]; then
+    DUPLICITY_JOB_OPTIONS=$DUPLICITY_JOB_OPTIONS" "${VOLUMERIZE_DUPLICITY_OPTIONS}
+  fi
+  if [ ! -n "${PASSPHRASE}" ] && [ ! -n "${VOLUMERIZE_GPG_PUBLIC_KEY}" ] && [ ! -n "${VOLUMERIZE_GPG_PRIVATE_KEY}" ]; then
+    DUPLICITY_JOB_OPTIONS=$DUPLICITY_JOB_OPTIONS" --no-encryption"
+  fi
+  if [ -n "${GPG_KEY_ID}" ]; then
+    DUPLICITY_JOB_OPTIONS=$DUPLICITY_JOB_OPTIONS' --gpg-options "--trust-model always"'" --encrypt-key ${GPG_KEY_ID}"
+  fi
+  if [ -n "${VOLUMERIZE_FULL_IF_OLDER_THAN}" ]; then
+    DUPLICITY_JOB_OPTIONS=$DUPLICITY_JOB_OPTIONS" --full-if-older-than ${VOLUMERIZE_FULL_IF_OLDER_THAN}"
+  fi
+  if [ "${VOLUMERIZE_ASYNCHRONOUS_UPLOAD}" = 'true' ]; then
+    DUPLICITY_JOB_OPTIONS=$DUPLICITY_JOB_OPTIONS" --asynchronous-upload"
+  fi
+}
+
+function prepareJobConfiguration() {
+  local jobNumber=$1
+  local VARIABLE_SOURCE="VOLUMERIZE_SOURCE${jobNumber}"
+  local VARIABLE_TARGET="VOLUMERIZE_TARGET${jobNumber}"
+
+  if [ -n "${!VARIABLE_SOURCE}" ]; then
+    VOLUMERIZE_JOB_SOURCE=${!VARIABLE_SOURCE}
+  else
+    VOLUMERIZE_JOB_SOURCE=
+  fi
+  if [ -n "${!VARIABLE_TARGET}" ]; then
+    VOLUMERIZE_JOB_TARGET=${!VARIABLE_TARGET}
+  else
+    VOLUMERIZE_JOB_TARGET=
+  fi
+}
+
+function resolveJobIncludes() {
+  local jobNumber=$1
+  local x
+  local VARIABLE_INCLUDE
+  VOLUMERIZE_JOB_INCUDES=
+  for (( x=1; ; x++ ))
+  do
+    VARIABLE_INCLUDE="VOLUMERIZE_INCLUDE${jobNumber}_${x}"
+    if [ ! -n "${!VARIABLE_INCLUDE}" ]; then
+      break
+    fi
+    VOLUMERIZE_JOB_INCUDES=$VOLUMERIZE_JOB_INCUDES" --include "${!VARIABLE_INCLUDE}
+  done
+}
+
+function prepareJob() {
+  local jobNumber=$1
+  JOB_VARIABLE="VOLUMERIZE_SOURCE${jobNumber}"
+  if [ -n "${!JOB_VARIABLE}" ]; then
+    prepareJobCommand $jobNumber
+    prepareJobConfiguration $jobNumber
+    resolveJobIncludes $jobNumber
+  fi
+}
+
 resolveIncludes
 resolveOptions
+discoverJobs

scripts/backup

@@ -8,13 +8,40 @@ source /opt/volumerize/base.sh
 
 DUPLICITY_RETURN_CODE=0
 
-function backup() {
-  ${DUPLICITY_COMMAND} $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ;
+function commandLoop() {
+  local jobcount=$JOB_COUNT
+  local counter;
+
+  for (( counter=1; counter<=$jobcount; counter++ ))
+  do
+    commandJob "$counter" $@
+  done
+}
+
+function commandJob() {
+  local jobNumber=$1
+  local returnCode=0;
+  prepareJob "$jobNumber"
+  shift;
+  ${DUPLICITY_JOB_COMMAND} $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_INCUDES} ${VOLUMERIZE_JOB_SOURCE} ${VOLUMERIZE_JOB_TARGET} || returnCode=$? && true ;
+  if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then
+    DUPLICITY_RETURN_CODE=$returnCode
+  fi
+}
+
+function commandExecution() {
+  if [ -n "${VOLUMERIZE_SOURCE}" ]; then
+    ${DUPLICITY_COMMAND} $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ;
+  elif [ ! -z "${1##*[!0-9]*}" ]; then
+    commandJob $@
+  elif [ -n "${JOB_COUNT}" ]; then
+    commandLoop $@
+  fi
 }
 
 ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction backup
-source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers.sh
-backup $@
-source ${VOLUMERIZE_SCRIPT_DIR}/startContainers.sh
+${VOLUMERIZE_SCRIPT_DIR}/stopContainers.sh
+commandExecution $@
+${VOLUMERIZE_SCRIPT_DIR}/startContainers.sh
 ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction backup
 exit $DUPLICITY_RETURN_CODE

scripts/backupFull

@@ -8,13 +8,40 @@ source /opt/volumerize/base.sh
 
 DUPLICITY_RETURN_CODE=0
 
-function backupFull() {
-  ${DUPLICITY_COMMAND} full $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ;
+function commandLoop() {
+  local jobcount=$JOB_COUNT
+  local counter;
+
+  for (( counter=1; counter<=$jobcount; counter++ ))
+  do
+    commandJob "$counter" $@
+  done
+}
+
+function commandJob() {
+  local jobNumber=$1
+  local returnCode=0;
+  prepareJob "$jobNumber"
+  shift;
+  ${DUPLICITY_JOB_COMMAND} full $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_INCUDES} ${VOLUMERIZE_JOB_SOURCE} ${VOLUMERIZE_JOB_TARGET} || returnCode=$? && true ;
+  if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then
+    DUPLICITY_RETURN_CODE=$returnCode
+  fi
+}
+
+function commandExecution() {
+  if [ -n "${VOLUMERIZE_SOURCE}" ]; then
+    ${DUPLICITY_COMMAND} full $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ;
+  elif [ ! -z "${1##*[!0-9]*}" ]; then
+    commandJob $@
+  elif [ -n "${JOB_COUNT}" ]; then
+    commandLoop $@
+  fi
 }
 
 ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction backup
 source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers.sh
-backupFull $@
+commandExecution $@
 source ${VOLUMERIZE_SCRIPT_DIR}/startContainers.sh
 ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction backup
 exit $DUPLICITY_RETURN_CODE

scripts/backupIncremental

@@ -8,13 +8,40 @@ source /opt/volumerize/base.sh
 
 DUPLICITY_RETURN_CODE=0
 
-function backupIncremental() {
-  ${DUPLICITY_COMMAND} incremental $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ;
+function commandLoop() {
+  local jobcount=$JOB_COUNT
+  local counter;
+
+  for (( counter=1; counter<=$jobcount; counter++ ))
+  do
+    commandJob "$counter" $@
+  done
+}
+
+function commandJob() {
+  local jobNumber=$1
+  local returnCode=0;
+  prepareJob "$jobNumber"
+  shift;
+  ${DUPLICITY_JOB_COMMAND} incremental $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_INCUDES} ${VOLUMERIZE_JOB_SOURCE} ${VOLUMERIZE_JOB_TARGET} || returnCode=$? && true ;
+  if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then
+    DUPLICITY_RETURN_CODE=$returnCode
+  fi
+}
+
+function commandExecution() {
+  if [ -n "${VOLUMERIZE_SOURCE}" ]; then
+    ${DUPLICITY_COMMAND} incremental $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ;
+  elif [ ! -z "${1##*[!0-9]*}" ]; then
+    commandJob $@
+  elif [ -n "${JOB_COUNT}" ]; then
+    commandLoop $@
+  fi
 }
 
 ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction backup
 source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers.sh
-backupIncremental $@
+commandExecution $@
 source ${VOLUMERIZE_SCRIPT_DIR}/startContainers.sh
 ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction backup
 exit $DUPLICITY_RETURN_CODE

scripts/cleanup

@@ -6,4 +6,38 @@ set -o errexit
 
 source /opt/volumerize/base.sh
 
-exec ${DUPLICITY_COMMAND} cleanup $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET}
+DUPLICITY_RETURN_CODE=0
+
+function commandLoop() {
+  local jobcount=$JOB_COUNT
+  local counter;
+
+  for (( counter=1; counter<=$jobcount; counter++ ))
+  do
+    commandJob "$counter" $@
+  done
+}
+
+function commandJob() {
+  local jobNumber=$1
+  local returnCode=0;
+  prepareJob "$jobNumber"
+  shift;
+  ${DUPLICITY_JOB_COMMAND} cleanup $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_INCUDES} ${VOLUMERIZE_JOB_TARGET} || returnCode=$? && true ;
+  if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then
+    DUPLICITY_RETURN_CODE=$returnCode
+  fi
+}
+
+function commandExecution() {
+  if [ -n "${VOLUMERIZE_SOURCE}" ]; then
+    ${DUPLICITY_COMMAND} cleanup $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET}  || DUPLICITY_RETURN_CODE=$? && true ;
+  elif [ ! -z "${1##*[!0-9]*}" ]; then
+    commandJob $@
+  elif [ -n "${JOB_COUNT}" ]; then
+    commandLoop $@
+  fi
+}
+
+commandExecution $@
+exit $DUPLICITY_RETURN_CODE

scripts/list

@@ -6,4 +6,38 @@ set -o errexit
 
 source /opt/volumerize/base.sh
 
-exec ${DUPLICITY_COMMAND} collection-status $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET}
+DUPLICITY_RETURN_CODE=0
+
+function commandLoop() {
+  local jobcount=$JOB_COUNT
+  local counter;
+
+  for (( counter=1; counter<=$jobcount; counter++ ))
+  do
+    commandJob "$counter" $@
+  done
+}
+
+function commandJob() {
+  local jobNumber=$1
+  local returnCode=0;
+  prepareJob "$jobNumber"
+  shift;
+  ${DUPLICITY_JOB_COMMAND} collection-status $@ ${DUPLICITY_JOB_OPTIONS} ${VOLUMERIZE_JOB_INCUDES} ${VOLUMERIZE_JOB_TARGET} || returnCode=$? && true ;
+  if [ "$returnCode" -gt "$DUPLICITY_RETURN_CODE" ]; then
+    DUPLICITY_RETURN_CODE=$returnCode
+  fi
+}
+
+function commandExecution() {
+  if [ -n "${VOLUMERIZE_SOURCE}" ]; then
+    ${DUPLICITY_COMMAND} collection-status $@ ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET} || DUPLICITY_RETURN_CODE=$? && true ;
+  elif [ ! -z "${1##*[!0-9]*}" ]; then
+    commandJob $@
+  elif [ -n "${JOB_COUNT}" ]; then
+    commandLoop $@
+  fi
+}
+
+commandExecution $@
+exit $DUPLICITY_RETURN_CODE