Try to fix travis CI

Removed SSH backend scheme example

.travis.yml

@@ -3,12 +3,8 @@ language: bash
 # Use container-based infrastructure for quicker build start-up
 sudo: false
 
-addons:
-  apt:
-    sources:
-    - debian-sid
-    packages:
-    - shellcheck
+before_script:
+    - shellcheck --version
 
 script:
     - shellcheck -e SC2034 duplicity-backup.sh duplicity-backup.conf.example

CHANGELOG

@@ -1,10 +1,22 @@
 vX.X.X (unreleased)
 ===================
-  * Placeholder for next release
+
+v1.6.0 (2018-05-22)
+===================
+  * Add support for Dropbox backend (by xmatthias)
+  * Fix issue #177: bug in `--cleanup` action (thanks smcgrat)
+
+v1.5.0 (2018-01-15)
+===================
+  * Add support for B2 backend (by harrim4n)
+
+v1.4.3 (2017-10-30)
+===================
+  * Add support for Telegram notifications (by GerardRibas)
 
 v1.4.2 (2017-10-30)
 ===================
-  * Remove VERSION file and hardcode it directly in the script.
+  * Remove VERSION file and hardcode it directly in the script. Fix issue #164 (thanks regelga)
 
 v1.4.1 (2017-10-04)
 ===================

README.md

@@ -68,23 +68,23 @@ You can clone the repository (which makes it easy to get future updates):
 
     git clone https://github.com/zertrin/duplicity-backup.sh.git duplicity-backup
 
-If you prefer the stable version do:
+If you prefer the stable (but old) version do:
 
     git checkout stable
 
-... or if you want the latest version (might still have bugs), then:
+... or if you want the normal version, then:
 
     git checkout master
 
-... or if you like living on the edge, you can stay at the development version which is automatically cloned.
+... or if you like living on the edge (or need the latest bugfixes), you can stay at the development version which is automatically cloned.
 
 Or just download the ZIP file:
 
 | Version       | Download link                                                     |
 |---------------|-------------------------------------------------------------------|
-| stable        | https://github.com/zertrin/duplicity-backup.sh/archive/stable.zip |
+| stable (old)  | https://github.com/zertrin/duplicity-backup.sh/archive/stable.zip |
 | normal        | https://github.com/zertrin/duplicity-backup.sh/archive/master.zip |
-| bleeding-edge | https://github.com/zertrin/duplicity-backup.sh/archive/dev.zip    |
+| latest        | https://github.com/zertrin/duplicity-backup.sh/archive/dev.zip    |
 
 ### 2. Configure the script
 
@@ -139,6 +139,9 @@ There are many ways to do so, here are some examples (adapt the path to your act
 For the [Amazon S3](https://aws.amazon.com/s3/) storage backend *`optional`*
 * [s3cmd](http://s3tools.org/s3cmd) *`optional`*
 
+For the [Backblaze B2](https://www.backblaze.com/b2) storage backend *`optional`*
+* [python-b2](https://pypi.python.org/pypi/b2) *`optional`*
+
 For the [Google Cloud Storage](https://cloud.google.com/storage/) storage backend *`optional`*
 * [boto](https://github.com/boto/boto) (may already have been installed with duplicity)
 * [gsutil](https://cloud.google.com/storage/docs/gsutil) *`optional`*

duplicity-backup.conf.example

@@ -2,7 +2,7 @@
 #
 # Copyright (c) 2008-2010 Damon Timm.
 # Copyright (c) 2010 Mario Santagiuliana.
-# Copyright (c) 2012-2016 Marc Gallet.
+# Copyright (c) 2012-2018 Marc Gallet.
 #
 # This program is free software: you can redistribute it and/or modify it under
 # the terms of the GNU General Public License as published by the Free Software
@@ -75,7 +75,7 @@
 # This can be / or somewhere else -- I use /home/ because all the
 # directories that I want to backup start with /home/.
 #
-ROOT="/home"
+ROOT='/home'
 
 # Set hostname for this duplicity instance, useful for e-mail reports
 #
@@ -108,13 +108,14 @@ DEST="s3+http://foobar-backup-bucket/backup-folder/"
 #DEST="ftpes://user[:password]@other.host[:port]/some_dir"
 #DEST="rsync://user@host.com[:port]//absolute_path"
 #DEST="scp://user[:password]@other.host[:port]/[/]some_dir"
-#DEST="ssh://user[:password]@other.host[:port]/[/]some_dir"
 #DEST="sftp://user[:password]@other.host[:port]/[/]some_dir"
 #DEST="file:///home/foobar_user_name/new-backup-test/"
 #DEST="imap[s]://user[:password]@host.com[/from_address_prefix]"
 #DEST="webdav[s]://user[:password]@other.host[:port]/some_dir"
 #DEST="gdocs://foobar_google_account/some_dir"
 #DEST="swift://foobar_swift_container/some_dir"
+#DEST="dpbx:///foobar_swift_container/some_dir"
+#DEST="b2://some_account_id[:some_application_key]@some_bucket_name/some_dir"
 
 
 # ------------------------------------------------------------------------------
@@ -128,7 +129,7 @@ DEST="s3+http://foobar-backup-bucket/backup-folder/"
 #    setting it in the backend url (which might be readable in the operating
 #    systems process listing to other users on the same machine)."
 #
-#FTP_PASSWORD="password"
+#FTP_PASSWORD='password'
 
 
 # ------------------------------------------------------------------------------
@@ -173,6 +174,13 @@ DEST="s3+http://foobar-backup-bucket/backup-folder/"
 #SWIFT_AUTHURL="foobar_swift_authurl"
 #SWIFT_AUTHVERSION="2"
 
+# ------------------------------------------------------------------------------
+# DROPBOX INFORMATION
+# ------------------------------------------------------------------------------
+# Uncomment these lines if you're using Dropbox
+#
+#DPBX_ACCESS_TOKEN="foobar_dropbox_access_token"
+
 
 # ------------------------------------------------------------------------------
 # INCLUDE LIST OF DIRECTORIES
@@ -182,15 +190,15 @@ DEST="s3+http://foobar-backup-bucket/backup-folder/"
 #
 # Here is an example with multiple locations:
 #
-#INCLIST=(  "/home/*/Documents" \
-#           "/home/*/Projects" \
-#           "/home/*/logs" \
-#           "/home/www/mysql-backups" \
+#INCLIST=(  '/home/*/Documents' \
+#           '/home/*/Projects' \
+#           '/home/*/logs' \
+#           '/home/www/mysql-backups' \
 #        )
 #
 # Simpler example with one location:
 
-INCLIST=( "/home/foobar_user_name/Documents/" )
+INCLIST=( '/home/foobar_user_name/Documents/' )
 
 
 # ------------------------------------------------------------------------------
@@ -202,18 +210,18 @@ INCLIST=( "/home/foobar_user_name/Documents/" )
 #
 # Here is an example with multiple locations:
 #
-#EXCLIST=(   "/home/*/Trash" \
-#            "/home/*/Projects/Completed" \
-#            "/**.DS_Store" \
-#            "/**Icon?" \
-#            "/**.AppleDouble" \
+#EXCLIST=(   '/home/*/Trash' \
+#            '/home/*/Projects/Completed' \
+#            '/**.DS_Store' \
+#            '/**Icon?' \
+#            '/**.AppleDouble' \
 #        )
 #
 # If you don't want to exclude anything, use EXCLIST=()
 #
 # Simpler example with one location. Adapt it to your needs.
 
-EXCLIST=( "/home/foobar_user_name/Documents/foobar-to-exclude" )
+EXCLIST=( '/home/foobar_user_name/Documents/foobar-to-exclude' )
 
 
 # ------------------------------------------------------------------------------
@@ -260,9 +268,9 @@ ENCRYPTION='yes'
 # (your backups will be encrypted with this password) or is used
 # for the "GPG_SIGN_KEY" (see below).
 # Comment out if you aren't using encryption
-# Note: if you have a '$' in your passphrase, escape it with a '\'
+# Note: if you have a ' in your passphrase, escape it accordingly.
 
-PASSPHRASE="foobar_gpg_passphrase"
+PASSPHRASE='foobar_gpg_passphrase'
 
 # Specify which GPG keys you would like to use (even if you have only one).
 # If you are running this from a cron, it is highly recommended to create separate
@@ -419,7 +427,7 @@ MAIL="mailx"     # default command for Linux mail
 # Third-party notification services. If NOTIFICATION_SERVICE is not provided, no
 # notifications will be sent.
 
-# Possible values for NOTIFICATION_SERVICE are: slack, pushover, ifttt
+# Possible values for NOTIFICATION_SERVICE are: slack, pushover, ifttt, telegram
 NOTIFICATION_SERVICE=""
 NOTIFICATION_FAILURE_ONLY="yes" # send notifications only if there was an error while creating backup
 
@@ -439,6 +447,10 @@ IFTTT_MAKER_EVENT="duplicity" # name the event to trigger at IFTTT Maker Channel
 IFTTT_HOOK_URL="https://maker.ifttt.com/trigger/$IFTTT_MAKER_EVENT/with/key/$IFTTT_KEY" # ONLY change this if IFTTT changes it
 IFTTT_VALUE2="" # general purpose value to pass to your maker channel (optional)
 
+# Provider: Telegram
+TELEGRAM_CHATID="" #Generate a Telegram bot following guide: https://core.telegram.org/bots#3-how-do-i-create-a-bot
+TELEGRAM_KEY=""
+
 # ------------------------------------------------------------------------------
 # TROUBLESHOOTING
 # ------------------------------------------------------------------------------

duplicity-backup.sh

@@ -2,7 +2,7 @@
 #
 # Copyright (c) 2008-2010 Damon Timm.
 # Copyright (c) 2010 Mario Santagiuliana.
-# Copyright (c) 2012-2017 Marc Gallet.
+# Copyright (c) 2012-2018 Marc Gallet.
 #
 # This program is free software: you can redistribute it and/or modify it under
 # the terms of the GNU General Public License as published by the Free Software
@@ -45,7 +45,7 @@ CONFIG="duplicity-backup.conf"
 # Script Happens Below This Line - Shouldn't Require Editing #
 ##############################################################
 
-DBSH_VERSION="v1.4.2"
+DBSH_VERSION="v1.6.0"
 
 # make a backup of stdout and stderr for later
 exec 6>&1
@@ -191,7 +191,7 @@ while getopts ":c:t:bfvelsqndhV-:" opt; do
           QUIET=1
         ;;
         dry-run)
-          DRY_RUN="--dry-run "
+          DRY_RUN="--dry-run"
         ;;
         debug)
           ECHO=$(which echo)
@@ -218,7 +218,7 @@ while getopts ":c:t:bfvelsqndhV-:" opt; do
     l) COMMAND="list-current-files";;
     s) COMMAND="collection-status";;
     q) QUIET=1;;
-    n) DRY_RUN="--dry-run ";; # dry run
+    n) DRY_RUN="--dry-run";; # dry run
     d) ECHO=$(which echo);; # debug
     h)
       usage
@@ -342,7 +342,13 @@ fi
 
 # ------------------------- Setting up variables ------------------------
 
-STATIC_OPTIONS="${DRY_RUN}${STATIC_OPTIONS}"
+if [ -n "${DRY_RUN}" ]; then
+  STATIC_OPTIONS="${DRY_RUN} ${STATIC_OPTIONS}"
+fi
+
+if [ -n "${STORAGECLASS}" ]; then
+  STATIC_OPTIONS="${STATIC_OPTIONS} ${STORAGECLASS}"
+fi
 
 SIGN_PASSPHRASE=${PASSPHRASE}
 
@@ -354,6 +360,7 @@ export SWIFT_USERNAME
 export SWIFT_PASSWORD
 export SWIFT_AUTHURL
 export SWIFT_AUTHVERSION
+export DPBX_ACCESS_TOKEN
 export PASSPHRASE
 export SIGN_PASSPHRASE
 
@@ -403,6 +410,9 @@ NO_S3CMD_CFG="WARNING: s3cmd is not configured, run 's3cmd --configure' \
 in order to retrieve remote file size information. Remote file \
 size information unavailable."
 
+NO_B2CMD="WARNING: b2 not found in PATH, remote file size information \
+unavailable. Is the python-b2 package installed?"
+
 README_TXT="In case you've long forgotten, this is a backup script that you used to backup some files (most likely remotely at Amazon S3). In order to restore these files, you first need to import your GPG private(s) key(s) (if you haven't already). The key(s) is/are in this directory and the following command(s) should do the trick:\n\nIf you were using the same key for encryption and signature:\n  gpg --allow-secret-key-import --import duplicity-backup-encryption-and-sign-secret.key.txt\nOr if you were using two separate keys for encryption and signature:\n  gpg --allow-secret-key-import --import duplicity-backup-encryption-secret.key.txt\n  gpg --allow-secret-key-import --import duplicity-backup-sign-secret.key.txt\n\nAfter your key(s) has/have been succesfully imported, you should be able to restore your files.\n\nGood luck!"
 
 if  [ "$(echo "${DEST}" | cut -c 1,2)" = "gs" ]; then
@@ -445,6 +455,22 @@ else
   DEST_IS_S3=false
 fi
 
+if  [ "$(echo "${DEST}" | cut -c 1,4)" = "dpbx" ]; then
+  DEST_IS_DPBX=true
+else
+  DEST_IS_DPBX=false
+fi
+
+if  [ "$(echo "${DEST}" | cut -c 1,2)" = "b2" ]; then
+  DEST_IS_B2=true
+  B2CMD="$(which b2)"
+  if [ ! -x "${B2CMD}" ]; then
+    echo "${NO_B2CMD}"; B2CMD_AVAIL=false
+  fi
+else
+  DEST_IS_B2=false
+fi
+
 config_sanity_fail()
 {
   EXPLANATION=$1
@@ -468,6 +494,8 @@ check_variables ()
   config_sanity_fail "An s3 DEST has been specified, but AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY have not been configured"
   [[ ( ${DEST_IS_GS} = true && (${GS_ACCESS_KEY_ID} = "foobar_gcs_key_id" || ${GS_SECRET_ACCESS_KEY} = "foobar_gcs_secret_id" )) ]] && \
   config_sanity_fail "A Google Cloud Storage DEST has been specified, but GS_ACCESS_KEY_ID or GS_SECRET_ACCESS_KEY have not been configured"
+  [[ ( ${DEST_IS_DPBX} = true && (${DPBX_ACCESS_TOKEN} = "foobar_dropbox_access_token" )) ]] && \
+  config_sanity_fail "A Dropbox DEST has been specified, but DPBX_ACCESS_TOKEN has not been configured"
   [[ ! -z "${INCEXCFILE}" && ! -f ${INCEXCFILE} ]] && config_sanity_fail "The specified INCEXCFILE ${INCEXCFILE} does not exists"
 }
 
@@ -543,21 +571,34 @@ email_logfile()
 
 send_notification()
 {
-  NOTIFICATION_CONTENT="duplicity-backup ${BACKUP_STATUS:-"ERROR"} [${HOSTNAME}] - \`${LOGFILE}\`"
   if [ ! -z "${NOTIFICATION_SERVICE}" ]; then
+    echo "-----------[ Notification Request ]-----------"
+    NOTIFICATION_CONTENT="duplicity-backup ${BACKUP_STATUS:-"ERROR"} [${HOSTNAME}] - \`${LOGFILE}\`"
+
     if [ "${NOTIFICATION_SERVICE}" = "slack" ]; then
       curl -X POST -H 'Content-type: application/json' --data "{\"text\": \"${NOTIFICATION_CONTENT}\", \"channel\": \"${SLACK_CHANNEL}\", \"username\": \"${SLACK_USERNAME}\", \"icon_emoji\": \":${SLACK_EMOJI}:\"}" "${SLACK_HOOK_URL}"
-      echo -e "Slack notification sent to channel ${SLACK_CHANNEL}"
     elif [ "${NOTIFICATION_SERVICE}" = "ifttt" ]; then
       curl -X POST -H 'Content-type: application/json' --data "{\"value1\": \"${NOTIFICATION_CONTENT}\", \"value2\": \"${IFTTT_VALUE2}\"}" "${IFTTT_HOOK_URL}"
-      echo -e "IFTTT notification sent to Maker channel event ${IFTTT_EVENT}"
     elif [ "${NOTIFICATION_SERVICE}" = "pushover" ]; then
       curl -s \
       -F "token=${PUSHOVER_TOKEN}" \
       -F "user=${PUSHOVER_USER}" \
       -F "message=${NOTIFICATION_CONTENT}" \
       https://api.pushover.net/1/messages
+    elif [ "${NOTIFICATION_SERVICE}" = "telegram" ]; then
+      curl -s --max-time 10 -d "chat_id=${TELEGRAM_CHATID}&disable_web_page_preview=1&text=${NOTIFICATION_CONTENT}" "https://api.telegram.org/bot${TELEGRAM_KEY}/sendMessage" >/dev/null
+    fi
+
+    echo -e "\n----------------------------------------------\n"
+
+    if [ "${NOTIFICATION_SERVICE}" = "slack" ]; then
+      echo -e "Slack notification sent to channel ${SLACK_CHANNEL}"
+    elif [ "${NOTIFICATION_SERVICE}" = "ifttt" ]; then
+      echo -e "IFTTT notification sent to Maker channel event ${IFTTT_EVENT}"
+    elif [ "${NOTIFICATION_SERVICE}" = "pushover" ]; then
       echo -e "Pushover notification sent"
+    elif [ "${NOTIFICATION_SERVICE}" = "telegram" ]; then
+      echo -e "Telegram notification sent"
     fi
   fi
 }
@@ -647,7 +688,7 @@ get_remote_file_size()
     ;;
     "fi")
       FRIENDLY_TYPE_NAME="File"
-      TMPDEST=$(echo "${DEST}" | cut -c 6-)
+      TMPDEST="${DEST#file://*}"
       SIZE=$(du -hs "${TMPDEST}" | awk '{print $1}')
     ;;
     "gs")
@@ -676,6 +717,33 @@ get_remote_file_size()
           fi
       fi
     ;;
+    "b2")
+      FRIENDLY_TYPE_NAME="Backblaze B2"
+      if ${B2CMD_AVAIL}; then
+        if [[ -n ${FTP_PASSWORD} ]]; then
+          APP_KEY=${FTP_PASSWORD}
+        else
+          APP_KEY=$(echo "${DEST}" | cut -d":" -f 3 | cut -d"@" -f 1)
+        fi
+        ACC_ID=$(echo "${DEST}" | cut -d"/" -f 3 | cut -d"@" -f 1 | cut -d ":" -f 1)
+        BUCKET=$(echo "${DEST}" | cut -d"@" -f2 | cut -d"/" -f1)
+        if [[ -z ${APP_KEY} ]] || [[ -z ${ACC_ID} ]]; then
+          SIZE="-b2 authentication wrong-"
+          return
+        fi
+        if [[ -z ${BUCKET} ]]; then
+          SIZE="-b2 bucket wrong-"
+          return
+        fi
+        if [[ $(${B2CMD} authorize-account "${ACC_ID}" "${APP_KEY}" >/dev/null 2>&1) -ne 0 ]]; then
+          SIZE="-b2 authentication wrong-"
+          return
+        fi
+        SIZE=$(${B2CMD} ls --long "${BUCKET}" | awk '{ print $5 }' | paste -sd+ | bc | numfmt --to=iec)
+      else
+              SIZE="-b2 not found in PATH-"
+      fi
+    ;;
     *)
       # not yet available for the other backends
       FRIENDLY_TYPE_NAME=""
@@ -686,7 +754,7 @@ get_remote_file_size()
       echo -e "${SIZE}\t${FRIENDLY_TYPE_NAME} type backend"
   else
       echo "Destination disk use information is currently only available for the following storage backends:"
-      echo "File, SSH, Amazon S3 and Google Cloud"
+      echo "File, SSH, Amazon S3, Google Cloud and Backblaze B2"
   fi
   echo
 }
@@ -772,7 +840,6 @@ duplicity_backup()
 {
   {
     eval "${ECHO}" "${DUPLICITY}" "${OPTION}" "${VERBOSITY}" "${STATIC_OPTIONS}" \
-    "${STORAGECLASS}" \
     "${ENCRYPT}" \
     "${EXCLUDE}" \
     "${INCLUDE}" \
@@ -787,6 +854,7 @@ duplicity_cleanup_failed()
 {
   {
     eval "${ECHO}" "${DUPLICITY}" "${OPTION}" "${VERBOSITY}" "${STATIC_OPTIONS}" \
+    "${ENCRYPT}" \
     "${DEST}"
   } || {
     BACKUP_ERROR=1
@@ -951,7 +1019,7 @@ case "${COMMAND}" in
     OPTION="cleanup"
 
     if [ -z "${DRY_RUN}" ]; then
-      STATIC_OPTIONS="--force"
+      STATIC_OPTIONS="${STATIC_OPTIONS} --force"
     fi
 
     echo -e "-------[ Cleaning up Destination ]-------\n"
@@ -1098,6 +1166,11 @@ unset AWS_ACCESS_KEY_ID
 unset AWS_SECRET_ACCESS_KEY
 unset GS_ACCESS_KEY_ID
 unset GS_SECRET_ACCESS_KEY
+unset SWIFT_USERNAME
+unset SWIFT_PASSWORD
+unset SWIFT_AUTHURL
+unset SWIFT_AUTHVERSION
+unset DPBX_ACCESS_TOKEN
 unset PASSPHRASE
 unset SIGN_PASSPHRASE
 unset FTP_PASSWORD