From 6256f0a7012416c9b9513287261c98fe1059c9e7 Mon Sep 17 00:00:00 2001 From: Giovanni Pini Date: Sun, 10 May 2015 11:18:33 +0200 Subject: [PATCH 1/4] Update README.md --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 32767ae..45e1367 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # duplicity-backup.sh -This bash script was designed to automate and simplify the remote backup process of [duplicity](http://duplicity.nongnu.org/) on Amazon S3 primarily. Other backup destinations are possible (FTP, SFTP, SCP, rsync, file...), i.e. any of duplicity's supported outputs. +This bash script was designed to automate and simplify the remote backup process of [duplicity](http://duplicity.nongnu.org/) on Amazon S3 primarily. Other backup destinations are possible (Google Cloud Storage, FTP, SFTP, SCP, rsync, file...), i.e. any of duplicity's supported outputs. After your script is configured, you can easily backup, restore, verify and clean (either via cron or manually) your data without having to remember lots of different command options and passphrases. @@ -49,10 +49,15 @@ Be sure to make the script executable (`chmod +x`) before you hit the gas. * [duplicity](http://duplicity.nongnu.org/) * Basic utilities like: [bash](https://www.gnu.org/software/bash/), [which](http://unixhelp.ed.ac.uk/CGI/man-cgi?which), [find](https://www.gnu.org/software/findutils/) and [tee](http://linux.die.net/man/1/tee) (should already be available on most Linux systems) * [gpg](https://www.gnupg.org/) *`optional`* -* [Amazon S3](https://aws.amazon.com/s3/) *`optional`* -* [s3cmd](http://s3tools.org/s3cmd) *`optional`* * [mailx](http://linux.die.net/man/1/mailx) *`optional`* +For [Amazon S3](https://aws.amazon.com/s3/) *`optional`* +* [s3cmd](http://s3tools.org/s3cmd) *`optional`* + +For [Google Cloud Storage](https://cloud.google.com/storage/) *`optional`* +* [boto](https://github.com/boto/boto) +* [gsutil](https://cloud.google.com/storage/docs/gsutil) *`optional`* + ## Configuration From 8abfd40c302fb64d03a961186672f7542959920a Mon Sep 17 00:00:00 2001 From: Giovanni Pini Date: Sun, 10 May 2015 12:49:41 +0200 Subject: [PATCH 2/4] Google Cloud Storage support --- duplicity-backup.conf.example | 6 ++++++ duplicity-backup.sh | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/duplicity-backup.conf.example b/duplicity-backup.conf.example index 6f27460..ef21eb4 100644 --- a/duplicity-backup.conf.example +++ b/duplicity-backup.conf.example @@ -46,6 +46,11 @@ AWS_ACCESS_KEY_ID="foobar_aws_key_id" AWS_SECRET_ACCESS_KEY="foobar_aws_access_key" +# GOOGLE CLOUD STORAGE INFORMATION +# Comment out these lines if you're not using Google Cloud storage +GS_ACCESS_KEY_ID="foobar_gcs_key_id" +GS_SECRET_ACCESS_KEY="foobar_gcs_secret_id" + # S3CMD INFORMATION # Most people don't need this, but in some cases # you may want to specify a custom configuration file @@ -117,6 +122,7 @@ DEST="s3+http://foobar-backup-bucket/backup-folder/" # Be sure to check duplicity's man page to know how to use them # (http://duplicity.nongnu.org/duplicity.1.html) # +#DEST="gs://foobar-backup-bucket/backup-folder/" #DEST="ftp://user[:password]@other.host[:port]/some_dir" #DEST="rsync://user@host.com[:port]//absolute_path" #DEST="scp://user[:password]@other.host[:port]/[/]some_dir" diff --git a/duplicity-backup.sh b/duplicity-backup.sh index 1a093f9..2affd82 100755 --- a/duplicity-backup.sh +++ b/duplicity-backup.sh @@ -171,6 +171,8 @@ STATIC_OPTIONS="$DRY_RUN$STATIC_OPTIONS" SIGN_PASSPHRASE=$PASSPHRASE export AWS_ACCESS_KEY_ID export AWS_SECRET_ACCESS_KEY +export GS_ACCESS_KEY_ID +export GS_SECRET_ACCESS_KEY export PASSPHRASE export SIGN_PASSPHRASE if [[ -n "$FTP_PASSWORD" ]]; then @@ -224,6 +226,10 @@ if [ ! -x "$DUPLICITY" ]; then exit 1 fi +if [ "`echo ${DEST} | cut -c 1,2`" = "gs" ]; then + DEST_IS_GS=true +fi + if [ "`echo ${DEST} | cut -c 1,2`" = "s3" ]; then DEST_IS_S3=true S3CMD="$(which s3cmd)" @@ -270,6 +276,8 @@ check_variables () [[ ${LOGDIR} = "/home/foobar_user_name/logs/test2/" ]] && config_sanity_fail "LOGDIR must be configured" [[ ( ${DEST_IS_S3} = true && (${AWS_ACCESS_KEY_ID} = "foobar_aws_key_id" || ${AWS_SECRET_ACCESS_KEY} = "foobar_aws_access_key" )) ]] && \ config_sanity_fail "An s3 DEST has been specified, but AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY have not been configured" + [[ ( ${DEST_IS_GS} = true && (${GS_ACCESS_KEY_ID} = "foobar_gcs_key_id" || ${GS_SECRET_ACCESS_KEY} = "foobar_gcs_secret_id" )) ]] && \ + config_sanity_fail "A Google Cloud Storage DEST has been specified, but GS_ACCESS_KEY_ID or GS_SECRET_ACCESS_KEY have not been configured" [[ ! -z "$INCEXCFILE" && ! -f $INCEXCFILE ]] && config_sanity_fail "The specified INCEXCFILE $INCEXCFILE does not exists" } @@ -406,6 +414,11 @@ get_remote_file_size() TMPDEST=`echo ${DEST} | cut -c 6-` SIZE=`du -hs ${TMPDEST} | awk '{print $1}'` ;; + "gs") + FRIENDLY_TYPE_NAME="Google Cloud Storage" + #TMPDEST=`echo ${DEST} | cut -c 4-` + #SIZE=`du -hs ${TMPDEST} | awk '{print $1}'` + ;; "s3") FRIENDLY_TYPE_NAME="S3" if $S3CMD_AVAIL ; then From 01143b6834cc22a5c137634553b7ad0a61c0bbcd Mon Sep 17 00:00:00 2001 From: Giovanni Pini Date: Mon, 11 May 2015 14:14:24 +0200 Subject: [PATCH 3/4] Size GCS with gsutil --- README.md | 5 +++++ duplicity-backup.sh | 24 ++++++++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 45e1367..9d64de9 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,11 @@ Be sure to make the script executable (`chmod +x`) before you hit the gas. * Basic utilities like: [bash](https://www.gnu.org/software/bash/), [which](http://unixhelp.ed.ac.uk/CGI/man-cgi?which), [find](https://www.gnu.org/software/findutils/) and [tee](http://linux.die.net/man/1/tee) (should already be available on most Linux systems) * [gpg](https://www.gnupg.org/) *`optional`* * [mailx](http://linux.die.net/man/1/mailx) *`optional`* +For [Amazon S3](https://aws.amazon.com/s3/) *`optional`* +* [s3cmd](http://s3tools.org/s3cmd) *`optional`* +For [Google Cloud Storage](https://cloud.google.com/storage/) *`optional`* +* [boto](https://github.com/boto/boto) +* [gsutil](https://cloud.google.com/storage/docs/gsutil) *`optional`* For [Amazon S3](https://aws.amazon.com/s3/) *`optional`* * [s3cmd](http://s3tools.org/s3cmd) *`optional`* diff --git a/duplicity-backup.sh b/duplicity-backup.sh index 2affd82..be29261 100755 --- a/duplicity-backup.sh +++ b/duplicity-backup.sh @@ -214,6 +214,11 @@ elif [ "$ENCRYPTION" = "no" ]; then ENCRYPT="--no-encryption" fi +NO_GSCMD="WARNING: gsutil no found in PATH, remote file \ +size information unavailable." +NO_GSCMD_CFG="WARNING: gsutil is not configured, run 'gsutil config' \ +in order to retrieve remote file size information. Remote file \ +size information unavailable." NO_S3CMD="WARNING: s3cmd no found in PATH, remote file \ size information unavailable." NO_S3CMD_CFG="WARNING: s3cmd is not configured, run 's3cmd --configure' \ @@ -228,6 +233,17 @@ fi if [ "`echo ${DEST} | cut -c 1,2`" = "gs" ]; then DEST_IS_GS=true + GSCMD="$(which gsutil)" + if [ ! -x "$GSCMD" ]; then + echo $NO_GSCMD; GSCMD_AVAIL=false + elif [ ! -f "${HOME}/.boto" ]; then + echo $NO_GSCMD_CFG; GSCMD_AVAIL=false + else + GSCMD_AVAIL=true + GSCMD="${GSCMD}" + fi +else + DEST_IS_GS=false fi if [ "`echo ${DEST} | cut -c 1,2`" = "s3" ]; then @@ -416,8 +432,10 @@ get_remote_file_size() ;; "gs") FRIENDLY_TYPE_NAME="Google Cloud Storage" - #TMPDEST=`echo ${DEST} | cut -c 4-` - #SIZE=`du -hs ${TMPDEST} | awk '{print $1}'` + if $GSCMD_AVAIL ; then + TMPDEST=`echo $DEST | sed -e "s/\/*$//" ` + SIZE=`gsutil du -s ${TMPDEST} | awk '{print $1}'` + fi ;; "s3") FRIENDLY_TYPE_NAME="S3" @@ -799,6 +817,8 @@ fi unset AWS_ACCESS_KEY_ID unset AWS_SECRET_ACCESS_KEY +unset GS_ACCESS_KEY_ID +unset GS_SECRET_ACCESS_KEY unset PASSPHRASE unset SIGN_PASSPHRASE unset FTP_PASSWORD From 2be958e0ec1002573955fd692531cfbe864a2b23 Mon Sep 17 00:00:00 2001 From: Giovanni Pini Date: Thu, 14 May 2015 14:19:43 +0200 Subject: [PATCH 4/4] Human readable size for Google Cloud --- duplicity-backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/duplicity-backup.sh b/duplicity-backup.sh index be29261..bdb23eb 100755 --- a/duplicity-backup.sh +++ b/duplicity-backup.sh @@ -434,7 +434,7 @@ get_remote_file_size() FRIENDLY_TYPE_NAME="Google Cloud Storage" if $GSCMD_AVAIL ; then TMPDEST=`echo $DEST | sed -e "s/\/*$//" ` - SIZE=`gsutil du -s ${TMPDEST} | awk '{print $1}'` + SIZE=`gsutil du -hs ${TMPDEST} | awk '{print $1$2}'` fi ;; "s3")