Merge branch 'bova82-google-cloud-storage' into dev

Conflicts:
	duplicity-backup.sh

README.md

@@ -1,6 +1,6 @@
 # duplicity-backup.sh
 
-This bash script was designed to automate and simplify the remote backup process of [duplicity](http://duplicity.nongnu.org/) on Amazon S3 primarily. Other backup destinations are possible (FTP, SFTP, SCP, rsync, file...), i.e. any of duplicity's supported outputs.
+This bash script was designed to automate and simplify the remote backup process of [duplicity](http://duplicity.nongnu.org/) on Amazon S3 primarily. Other backup destinations are possible (Google Cloud Storage, FTP, SFTP, SCP, rsync, file...), i.e. any of duplicity's supported outputs.
 
 After your script is configured, you can easily backup, restore, verify and clean (either via cron or manually) your data without having to remember lots of different command options and passphrases.
 
@@ -49,9 +49,19 @@ Be sure to make the script executable (`chmod +x`) before you hit the gas.
 * [duplicity](http://duplicity.nongnu.org/)
 * Basic utilities like: [bash](https://www.gnu.org/software/bash/), [which](http://unixhelp.ed.ac.uk/CGI/man-cgi?which), [find](https://www.gnu.org/software/findutils/) and [tee](http://linux.die.net/man/1/tee) (should already be available on most Linux systems)
 * [gpg](https://www.gnupg.org/) *`optional`*
-* [Amazon S3](https://aws.amazon.com/s3/) *`optional`*
-* [s3cmd](http://s3tools.org/s3cmd) *`optional`*
 * [mailx](http://linux.die.net/man/1/mailx) *`optional`*
+For [Amazon S3](https://aws.amazon.com/s3/) *`optional`*
+* [s3cmd](http://s3tools.org/s3cmd) *`optional`*
+For [Google Cloud Storage](https://cloud.google.com/storage/) *`optional`*
+* [boto](https://github.com/boto/boto) 
+* [gsutil](https://cloud.google.com/storage/docs/gsutil) *`optional`*
+
+For [Amazon S3](https://aws.amazon.com/s3/) *`optional`*
+* [s3cmd](http://s3tools.org/s3cmd) *`optional`*
+
+For [Google Cloud Storage](https://cloud.google.com/storage/) *`optional`*
+* [boto](https://github.com/boto/boto) 
+* [gsutil](https://cloud.google.com/storage/docs/gsutil) *`optional`*
 
 
 ## Configuration

duplicity-backup.conf.example

@@ -46,6 +46,11 @@
 AWS_ACCESS_KEY_ID="foobar_aws_key_id"
 AWS_SECRET_ACCESS_KEY="foobar_aws_access_key"
 
+# GOOGLE CLOUD STORAGE INFORMATION
+# Comment out these lines if you're not using Google Cloud storage
+GS_ACCESS_KEY_ID="foobar_gcs_key_id"
+GS_SECRET_ACCESS_KEY="foobar_gcs_secret_id"
+
 # S3CMD INFORMATION
 # Most people don't need this, but in some cases
 # you may want to specify a custom configuration file
@@ -117,6 +122,7 @@ DEST="s3+http://foobar-backup-bucket/backup-folder/"
 # Be sure to check duplicity's man page to know how to use them
 # (http://duplicity.nongnu.org/duplicity.1.html)
 #
+#DEST="gs://foobar-backup-bucket/backup-folder/"
 #DEST="ftp://user[:password]@other.host[:port]/some_dir"
 #DEST="rsync://user@host.com[:port]//absolute_path"
 #DEST="scp://user[:password]@other.host[:port]/[/]some_dir"

duplicity-backup.sh

@@ -171,6 +171,8 @@ STATIC_OPTIONS="$DRY_RUN$STATIC_OPTIONS"
 SIGN_PASSPHRASE=$PASSPHRASE
 export AWS_ACCESS_KEY_ID
 export AWS_SECRET_ACCESS_KEY
+export GS_ACCESS_KEY_ID
+export GS_SECRET_ACCESS_KEY
 export PASSPHRASE
 export SIGN_PASSPHRASE
 if [[ -n "$FTP_PASSWORD" ]]; then
@@ -212,6 +214,11 @@ elif [ "$ENCRYPTION" = "no" ]; then
   ENCRYPT="--no-encryption"
 fi
 
+NO_GSCMD="WARNING: gsutil not found in PATH, remote file \
+size information unavailable."
+NO_GSCMD_CFG="WARNING: gsutil is not configured, run 'gsutil config' \
+in order to retrieve remote file size information. Remote file \
+size information unavailable."
 NO_S3CMD="WARNING: s3cmd not found in PATH, remote file \
 size information unavailable."
 NO_S3CMD_CFG="WARNING: s3cmd is not configured, run 's3cmd --configure' \
@@ -224,6 +231,21 @@ if [ ! -x "$DUPLICITY" ]; then
   exit 1
 fi
 
+if  [ "`echo ${DEST} | cut -c 1,2`" = "gs" ]; then
+  DEST_IS_GS=true
+  GSCMD="$(which gsutil)"
+  if [ ! -x "$GSCMD" ]; then
+    echo $NO_GSCMD; GSCMD_AVAIL=false
+  elif [ ! -f "${HOME}/.boto" ]; then
+    echo $NO_GSCMD_CFG; GSCMD_AVAIL=false
+  else
+    GSCMD_AVAIL=true
+    GSCMD="${GSCMD}"
+  fi
+else
+  DEST_IS_GS=false
+fi
+
 if  [ "`echo ${DEST} | cut -c 1,2`" = "s3" ]; then
   DEST_IS_S3=true
   S3CMD="$(which s3cmd)"
@@ -270,6 +292,8 @@ check_variables ()
   [[ ${LOGDIR} = "/home/foobar_user_name/logs/test2/" ]] && config_sanity_fail "LOGDIR must be configured"
   [[ ( ${DEST_IS_S3} = true && (${AWS_ACCESS_KEY_ID} = "foobar_aws_key_id" || ${AWS_SECRET_ACCESS_KEY} = "foobar_aws_access_key" )) ]] && \
   config_sanity_fail "An s3 DEST has been specified, but AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY have not been configured"
+  [[ ( ${DEST_IS_GS} = true && (${GS_ACCESS_KEY_ID} = "foobar_gcs_key_id" || ${GS_SECRET_ACCESS_KEY} = "foobar_gcs_secret_id" )) ]] && \
+  config_sanity_fail "A Google Cloud Storage DEST has been specified, but GS_ACCESS_KEY_ID or GS_SECRET_ACCESS_KEY have not been configured"
   [[ ! -z "$INCEXCFILE" && ! -f $INCEXCFILE ]] && config_sanity_fail "The specified INCEXCFILE $INCEXCFILE does not exists"
 }
 
@@ -414,6 +438,13 @@ get_remote_file_size()
       TMPDEST=`echo ${DEST} | cut -c 6-`
       SIZE=`du -hs ${TMPDEST} | awk '{print $1}'`
     ;;
+    "gs")
+      FRIENDLY_TYPE_NAME="Google Cloud Storage"
+      if $GSCMD_AVAIL ; then
+        TMPDEST=`echo $DEST | sed -e "s/\/*$//" `
+        SIZE=`gsutil du -hs ${TMPDEST} | awk '{print $1$2}'`
+      fi
+    ;;
     "s3")
       FRIENDLY_TYPE_NAME="S3"
       if $S3CMD_AVAIL ; then
@@ -794,6 +825,8 @@ fi
 
 unset AWS_ACCESS_KEY_ID
 unset AWS_SECRET_ACCESS_KEY
+unset GS_ACCESS_KEY_ID
+unset GS_SECRET_ACCESS_KEY
 unset PASSPHRASE
 unset SIGN_PASSPHRASE
 unset FTP_PASSWORD