2Shirt/duplicity-backup.sh
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rework of the script to include some github patches that were brought to the original script in order to make the script more generic than only Amazon S3 oriented and fix a few issues.

Browse source
This commit is contained in:
Zertrin 2012-05-05 00:14:13 +02:00
parent d61caa1b7d
commit 32a241c2a6
1 changed files with 229 additions and 157 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

386
dt-s3-backup.sh
View file

@ -2,6 +2,7 @@
# #
# Copyright (c) 2008-2010 Damon Timm. # Copyright (c) 2008-2010 Damon Timm.
# Copyright (c) 2010 Mario Santagiuliana. # Copyright (c) 2010 Mario Santagiuliana.
# Copyright (c) 2012 Marc Gallet.
# #
# This program is free software: you can redistribute it and/or modify it under # This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU General Public License as published by the Free Software # the terms of the GNU General Public License as published by the Free Software
@ -18,19 +19,24 @@
# #
# MORE ABOUT THIS SCRIPT AVAILABLE IN THE README AND AT: # MORE ABOUT THIS SCRIPT AVAILABLE IN THE README AND AT:
# #
# http://damontimm.com/code/dt-s3-backup # http://zertrin.org/duplicity-backup.html (for this version)
# http://damontimm.com/code/dt-s3-backup (for the original programi by Damon Timm)
#
# Latest code available at:
# http://github.com/zertrin/duplicity-backup
# #
# ---------------------------------------------------------------------------- # # ---------------------------------------------------------------------------- #
# AMAZON S3 INFORMATION # AMAZON S3 INFORMATION
# Comment out this lines if you're not using S3 # Comment out this lines if you're not using S3
export AWS_ACCESS_KEY_ID="foobar_aws_key_id" AWS_ACCESS_KEY_ID="foobar_aws_key_id"
export AWS_SECRET_ACCESS_KEY="foobar_aws_access_key" AWS_SECRET_ACCESS_KEY="foobar_aws_access_key"
# ENCRYPTION INFORMATION
# If you aren't running this from a cron, comment this line out # If you aren't running this from a cron, comment this line out
# and duplicity should prompt you for your password. # and duplicity should prompt you for your password.
# Comment out if you're not using encryption # Comment out if you're not using encryption
export PASSPHRASE="foobar_gpg_passphrase" PASSPHRASE="foobar_gpg_passphrase"
# Specify which GPG key you would like to use (even if you have only one). # Specify which GPG key you would like to use (even if you have only one).
# Comment out if you're not using encryption # Comment out if you're not using encryption
@ -39,6 +45,7 @@ GPG_KEY="foobar_gpg_key"
# Do you want your backup to be encrypted? yes/no # Do you want your backup to be encrypted? yes/no
ENCRYPTION='yes' ENCRYPTION='yes'
# BACKUP SOURCE INFORMATION
# The ROOT of your backup (where you want the backup to start); # The ROOT of your backup (where you want the backup to start);
# This can be / or somwhere else -- I use /home/ because all the # This can be / or somwhere else -- I use /home/ because all the
# directories start with /home/ that I want to backup. # directories start with /home/ that I want to backup.
@ -52,13 +59,12 @@ ROOT="/home"
# #
# NOTE: You do need to keep the "s3+http://<your location>/" format # NOTE: You do need to keep the "s3+http://<your location>/" format
# even though duplicity supports "s3://<your location>/". # even though duplicity supports "s3://<your location>/".
#DEST="s3+http://backup-bucket/backup-folder/" DEST="s3+http://backup-bucket/backup-folder/"
# Other possible locations # Other possible locations
#DEST="ftp://user[:password]@other.host[:port]/some_dir" #DEST="ftp://user[:password]@other.host[:port]/some_dir"
#DEST="rsync://user@host.com[:port]//absolute_path" #DEST="rsync://user@host.com[:port]//absolute_path"
#DEST="ssh://user[:password]@other.host[:port]/[/]some_dir" #DEST="ssh://user[:password]@other.host[:port]/[/]some_dir"
DEST="file:///home/foobar_user_name/new-backup-test/" #DEST="file:///home/foobar_user_name/new-backup-test/"
# INCLUDE LIST OF DIRECTORIES # INCLUDE LIST OF DIRECTORIES
# Here is a list of directories to include; if you want to include # Here is a list of directories to include; if you want to include
@ -137,6 +143,34 @@ EMAIL_SUBJECT=
############################################################## ##############################################################
# Script Happens Below This Line - Shouldn't Require Editing # # Script Happens Below This Line - Shouldn't Require Editing #
############################################################## ##############################################################
# Read config file
CONFIG=
while :
do
case $1 in
-c | --config)
CONFIG=$2
shift 2
;;
*)
break
;;
esac
done
if [ ! -z "$CONFIG" -a -f "$CONFIG" ];
then
. $CONFIG
elif [ ! -z "$CONFIG" -a ! -f "$CONFIG" ];
then
echo "ERROR: can't find config file!" >&2
fi
export AWS_ACCESS_KEY_ID
export AWS_SECRET_ACCESS_KEY
export PASSPHRASE
LOGFILE="${LOGDIR}${LOG_FILE}" LOGFILE="${LOGDIR}${LOG_FILE}"
DUPLICITY="$(which duplicity)" DUPLICITY="$(which duplicity)"
S3CMD="$(which s3cmd)" S3CMD="$(which s3cmd)"
@ -153,7 +187,7 @@ size information unavailable."
NO_S3CMD_CFG="WARNING: s3cmd is not configured, run 's3cmd --configure' \ NO_S3CMD_CFG="WARNING: s3cmd is not configured, run 's3cmd --configure' \
in order to retrieve remote file size information. Remote file \ in order to retrieve remote file size information. Remote file \
size information unavailable." size information unavailable."
README_TXT="In case you've long forgotten, this is a backup script that you used to backup some files (most likely remotely at Amazon S3). In order to restore these files, you first need to import your GPG private key (if you haven't already). The key is in this directory and the following command should do the trick:\n\ngpg --allow-secret-key-import --import s3-secret.key.txt\n\nAfter your key as been succesfully imported, you should be able to restore your files.\n\nGood luck!" README_TXT="In case you've long forgotten, this is a backup script that you used to backup some files (most likely remotely at Amazon S3). In order to restore these files, you first need to import your GPG private key (if you haven't already). The key is in this directory and the following command should do the trick:\n\ngpg --allow-secret-key-import --import dt-s3-backup-secret.key.txt\n\nAfter your key as been succesfully imported, you should be able to restore your files.\n\nGood luck!"
CONFIG_VAR_MSG="Oops!! ${0} was unable to run!\nWe are missing one or more important variables at the top of the script.\nCheck your configuration because it appears that something has not been set yet." CONFIG_VAR_MSG="Oops!! ${0} was unable to run!\nWe are missing one or more important variables at the top of the script.\nCheck your configuration because it appears that something has not been set yet."
if [ ! -x "$DUPLICITY" ]; then if [ ! -x "$DUPLICITY" ]; then
@ -188,41 +222,65 @@ get_source_file_size()
{ {
echo "---------[ Source File Size Information ]---------" >> ${LOGFILE} echo "---------[ Source File Size Information ]---------" >> ${LOGFILE}
# Patches to support spaces in paths-
# Remove space as a field separator temporarily
OLDIFS=$IFS
IFS=$(echo -en "\t\n")
for exclude in ${EXCLIST[@]}; do for exclude in ${EXCLIST[@]}; do
DUEXCLIST="${DUEXCLIST}${exclude}\n" DUEXCLIST="${DUEXCLIST}${exclude}\n"
done done
for include in ${INCLIST[@]} for include in ${INCLIST[@]}
do do
echo -e $DUEXCLIST | \ echo -e '"'$DUEXCLIST'"' | \
du -hs --exclude-from="-" ${include} | \ du -hs --exclude-from="-" ${include} | \
awk '{ print $2"\t"$1 }' \ awk '{ FS="\t"; $0=$0; print $1"\t"$2 }' \
>> ${LOGFILE} >> ${LOGFILE}
done done
echo >> ${LOGFILE} echo >> ${LOGFILE}
# Restore IFS
IFS=$OLDIFS
} }
get_remote_file_size() get_remote_file_size()
{ {
echo "------[ Destination File Size Information ]------" >> ${LOGFILE} echo "------[ Destination File Size Information ]------" >> ${LOGFILE}
if [ `echo ${DEST} | cut -c 1,2` = "fi" ]; then
TMPDEST=`echo ${DEST} | cut -c 6-` dest_type=`echo ${DEST} | cut -c 1,2`
SIZE=`du -hs ${TMPDEST} | awk '{print $1}'` case $dest_type in
elif [ `echo ${DEST} | cut -c 1,2` = "s3" ] && $S3CMD_AVAIL ; then "fi")
TMPDEST=$(echo ${DEST} | cut -c 11-) TMPDEST=`echo ${DEST} | cut -c 6-`
SIZE=`s3cmd du -H s3://${TMPDEST} | awk '{print $1}'` SIZE=`du -hs ${TMPDEST} | awk '{print $1}'`
else ;;
SIZE="s3cmd not installed." "s3")
fi if $S3CMD_AVAIL ; then
TMPDEST=$(echo ${DEST} | cut -c 11-)
SIZE=`s3cmd du -H s3://${TMPDEST} | awk '{print $1}'`
else
SIZE="s3cmd not installed."
fi
;;
*)
SIZE="Information on remote file size unavailable."
;;
esac
echo "Current Remote Backup File Size: ${SIZE}" >> ${LOGFILE} echo "Current Remote Backup File Size: ${SIZE}" >> ${LOGFILE}
echo >> ${LOGFILE} echo >> ${LOGFILE}
} }
include_exclude() include_exclude()
{ {
# Changes to handle spaces in directory names and filenames
# and wrapping the files to include and exclude in quotes.
OLDIFS=$IFS
IFS=$(echo -en "\t\n")
for include in ${INCLIST[@]} for include in ${INCLIST[@]}
do do
TMP=" --include="$include TMP=" --include=""'"$include"'"
INCLUDE=$INCLUDE$TMP INCLUDE=$INCLUDE$TMP
done done
for exclude in ${EXCLIST[@]} for exclude in ${EXCLIST[@]}
@ -231,20 +289,23 @@ include_exclude()
EXCLUDE=$EXCLUDE$TMP EXCLUDE=$EXCLUDE$TMP
done done
EXCLUDEROOT="--exclude=**" EXCLUDEROOT="--exclude=**"
# Restore IFS
IFS=$OLDIFS
} }
duplicity_cleanup() duplicity_cleanup()
{ {
echo "-----------[ Duplicity Cleanup ]-----------" >> ${LOGFILE} echo "-----------[ Duplicity Cleanup ]-----------" >> ${LOGFILE}
${ECHO} ${DUPLICITY} ${CLEAN_UP_TYPE} ${CLEAN_UP_VARIABLE} --force \ eval ${ECHO} ${DUPLICITY} ${CLEAN_UP_TYPE} ${CLEAN_UP_VARIABLE} ${STATIC_OPTIONS} --force \
${ENCRYPT} \ ${ENCRYPT} \
${DEST} >> ${LOGFILE} ${DEST} >> ${LOGFILE}
echo >> ${LOGFILE} echo >> ${LOGFILE}
} }
duplicity_backup() duplicity_backup()
{ {
${ECHO} ${DUPLICITY} ${OPTION} ${VERBOSITY} ${STATIC_OPTIONS} \ eval ${ECHO} ${DUPLICITY} ${OPTION} ${VERBOSITY} ${STATIC_OPTIONS} \
${ENCRYPT} \ ${ENCRYPT} \
${EXCLUDE} \ ${EXCLUDE} \
${INCLUDE} \ ${INCLUDE} \
@ -258,7 +319,7 @@ get_file_sizes()
get_source_file_size get_source_file_size
get_remote_file_size get_remote_file_size
sed -i '/-------------------------------------------------/d' ${LOGFILE} sed -i -e '/^--*$/d' ${LOGFILE}
chown ${LOG_FILE_OWNER} ${LOGFILE} chown ${LOG_FILE_OWNER} ${LOGFILE}
} }
@ -288,7 +349,7 @@ backup_this_script()
mkdir -p ${TMPDIR} mkdir -p ${TMPDIR}
cp $SCRIPTPATH ${TMPDIR}/ cp $SCRIPTPATH ${TMPDIR}/
gpg -a --export-secret-keys ${GPG_KEY} > ${TMPDIR}/s3-secret.key.txt gpg -a --export-secret-keys ${GPG_KEY} > ${TMPDIR}/dt-s3-backup-secret.key.txt
echo -e ${README_TXT} > ${README} echo -e ${README_TXT} > ${README}
echo "Encrypting tarball, choose a password you'll remember..." echo "Encrypting tarball, choose a password you'll remember..."
tar c ${TMPDIR} | gpg -aco ${TMPFILENAME} tar c ${TMPDIR} | gpg -aco ${TMPFILENAME}
@ -312,161 +373,172 @@ check_variables ()
fi fi
} }
echo -e "-------- START DT-S3-BACKUP SCRIPT --------\n" >> ${LOGFILE} echo -e "-------- START dt-s3-backup SCRIPT --------\n" >> ${LOGFILE}
if [ "$1" = "--backup-script" ]; then case "$1" in
backup_this_script "--backup-script")
exit backup_this_script
elif [ "$1" = "--full" ]; then exit
check_variables ;;
OPTION="full"
include_exclude
duplicity_backup
duplicity_cleanup
get_file_sizes
elif [ "$1" = "--verify" ]; then "--full")
check_variables check_variables
OLDROOT=${ROOT} OPTION="full"
ROOT=${DEST} include_exclude
DEST=${OLDROOT} duplicity_backup
OPTION="verify" duplicity_cleanup
get_file_sizes
;;
echo -e "-------[ Verifying Source & Destination ]-------\n" >> ${LOGFILE} "--verify")
include_exclude check_variables
duplicity_backup OLDROOT=${ROOT}
ROOT=${DEST}
DEST=${OLDROOT}
OPTION="verify"
OLDROOT=${ROOT} echo -e "-------[ Verifying Source & Destination ]-------\n" >> ${LOGFILE}
ROOT=${DEST} include_exclude
DEST=${OLDROOT} duplicity_backup
get_file_sizes OLDROOT=${ROOT}
ROOT=${DEST}
DEST=${OLDROOT}
echo -e "Verify complete. Check the log file for results:\n>> ${LOGFILE}" get_file_sizes
elif [ "$1" = "--restore" ]; then echo -e "Verify complete. Check the log file for results:\n>> ${LOGFILE}"
check_variables ;;
ROOT=$DEST
OPTION="restore"
if [[ ! "$2" ]]; then "--restore")
echo "Please provide a destination path (eg, /home/user/dir):" check_variables
read -e NEWDESTINATION ROOT=$DEST
DEST=$NEWDESTINATION OPTION="restore"
echo ">> You will restore from ${ROOT} to ${DEST}"
echo "Are you sure you want to do that ('yes' to continue)?"
read ANSWER
if [[ "$ANSWER" != "yes" ]]; then
echo "You said << ${ANSWER} >> so I am exiting now."
echo -e "User aborted restore process ...\n" >> ${LOGFILE}
exit 1
fi
else
DEST=$2
fi
echo "Attempting to restore now ..." if [[ ! "$2" ]]; then
duplicity_backup echo "Please provide a destination path (eg, /home/user/dir):"
read -e NEWDESTINATION
DEST=$NEWDESTINATION
echo ">> You will restore from ${ROOT} to ${DEST}"
echo "Are you sure you want to do that ('yes' to continue)?"
read ANSWER
if [[ "$ANSWER" != "yes" ]]; then
echo "You said << ${ANSWER} >> so I am exiting now."
echo -e "User aborted restore process ...\n" >> ${LOGFILE}
exit 1
fi
else
DEST=$2
fi
elif [ "$1" = "--restore-file" ]; then echo "Attempting to restore now ..."
check_variables duplicity_backup
ROOT=$DEST ;;
INCLUDE=
EXCLUDE=
EXLUDEROOT=
OPTION=
if [[ ! "$2" ]]; then "--restore-file")
echo "Which file do you want to restore (eg, mail/letter.txt):" check_variables
read -e FILE_TO_RESTORE ROOT=$DEST
FILE_TO_RESTORE=$FILE_TO_RESTORE INCLUDE=
echo EXCLUDE=
else EXLUDEROOT=
FILE_TO_RESTORE=$2 OPTION=
fi
if [[ "$3" ]]; then if [[ ! "$2" ]]; then
DEST=$3 echo "Which file do you want to restore (eg, mail/letter.txt):"
else read -e FILE_TO_RESTORE
DEST=$(basename $FILE_TO_RESTORE) FILE_TO_RESTORE="'"$FILE_TO_RESTORE"'"
fi echo
else
FILE_TO_RESTORE="'"$2"'"
fi
echo -e "YOU ARE ABOUT TO..." if [[ "$3" ]]; then
echo -e ">> RESTORE: $FILE_TO_RESTORE" DEST="'"$3"'"
echo -e ">> TO: ${DEST}" else
echo -e "\nAre you sure you want to do that ('yes' to continue)?" DEST=$(basename $FILE_TO_RESTORE)
read ANSWER fi
if [ "$ANSWER" != "yes" ]; then
echo "You said << ${ANSWER} >> so I am exiting now." echo -e "YOU ARE ABOUT TO..."
echo -e ">> RESTORE: $FILE_TO_RESTORE"
echo -e ">> TO: ${DEST}"
echo -e "\nAre you sure you want to do that ('yes' to continue)?"
read ANSWER
if [ "$ANSWER" != "yes" ]; then
echo "You said << ${ANSWER} >> so I am exiting now."
echo -e "-------- END --------\n" >> ${LOGFILE}
exit 1
fi
echo "Restoring now ..."
#use INCLUDE variable without create another one
INCLUDE="--file-to-restore ${FILE_TO_RESTORE}"
duplicity_backup
;;
"--list-current-files")
check_variables
OPTION="list-current-files"
${DUPLICITY} ${OPTION} ${VERBOSITY} ${STATIC_OPTIONS} \
$ENCRYPT \
${DEST}
echo -e "-------- END --------\n" >> ${LOGFILE} echo -e "-------- END --------\n" >> ${LOGFILE}
exit 1 ;;
fi
echo "Restoring now ..." "--collection-status")
#use INCLUDE variable without create another one check_variables
INCLUDE="--file-to-restore ${FILE_TO_RESTORE}" OPTION="collection-status"
duplicity_backup ${DUPLICITY} ${OPTION} ${VERBOSITY} ${STATIC_OPTIONS} \
$ENCRYPT \
${DEST}
echo -e "-------- END --------\n" >> ${LOGFILE}
;;
elif [ "$1" = "--list-current-files" ]; then "--backup")
check_variables check_variables
OPTION="list-current-files" include_exclude
${DUPLICITY} ${OPTION} ${VERBOSITY} ${STATIC_OPTIONS} \ duplicity_backup
$ENCRYPT \ duplicity_cleanup
${DEST} get_file_sizes
echo -e "-------- END --------\n" >> ${LOGFILE} ;;
elif [ "$1" = "--collection-status" ]; then *)
check_variables echo -e "[Only show `basename $0` usage options]\n" >> ${LOGFILE}
OPTION="collection-status" echo " USAGE:
${DUPLICITY} ${OPTION} ${VERBOSITY} ${STATIC_OPTIONS} \ `basename $0` [-c configfile] [options]
$ENCRYPT \
${DEST}
echo -e "-------- END --------\n" >> ${LOGFILE}
elif [ "$1" = "--backup" ]; then Options:
check_variables --backup: runs an incremental backup
include_exclude --full: forces a full backup
duplicity_backup
duplicity_cleanup
get_file_sizes
else --verify: verifies the backup
echo -e "[Only show `basename $0` usage options]\n" >> ${LOGFILE} --restore [path]: restores the entire backup
echo " USAGE: --restore-file [file] [destination/filename]: restore a specific file
`basename $0` [options] --list-current-files: lists the files currently backed up in the archive
--collection-status: show all the backup sets in the archive
Options: --backup-script: automatically backup the script and secret key to the current working directory
--backup: runs an incremental backup
--full: forces a full backup
--verify: verifies the backup CURRENT SCRIPT VARIABLES:
--restore [path]: restores the entire backup ========================
--restore-file [file] [destination/filename]: restore a specific file DEST (backup destination) = ${DEST}
--list-current-files: lists the files currently backed up in the archive INCLIST (directories included) = ${INCLIST[@]:0}
--collection-status: show all the backup sets in the archive EXCLIST (directories excluded) = ${EXCLIST[@]:0}
ROOT (root directory of backup) = ${ROOT}
"
;;
esac
--backup-script: automatically backup the script and secret key to the current working directory echo -e "-------- END dt-s3-backup SCRIPT --------\n" >> ${LOGFILE}
CURRENT SCRIPT VARIABLES:
========================
DEST (backup destination) = ${DEST}
INCLIST (directories included) = ${INCLIST[@]:0}
EXCLIST (directories excluded) = ${EXCLIST[@]:0}
ROOT (root directory of backup) = ${ROOT}
"
fi
echo -e "-------- END DT-S3-BACKUP SCRIPT --------\n" >> ${LOGFILE}
if [ $EMAIL_TO ]; then if [ $EMAIL_TO ]; then
if [ ! -x "$MAIL" ]; then if [ ! -x "$MAIL" ]; then
echo -e "Email coulnd't be sent. mailx not available." >> ${LOGFILE} echo -e "Email couldn't be sent. mailx not available." >> ${LOGFILE}
else else
EMAIL_FROM=${EMAIL_FROM:+"-r ${EMAIL_FROM}"} EMAIL_FROM=${EMAIL_FROM:+"-r ${EMAIL_FROM}"}
EMAIL_SUBJECT=${EMAIL_SUBJECT:="DT-S3 Alert ${LOG_FILE}"} EMAIL_SUBJECT=${EMAIL_SUBJECT:="DT-S3 Alert ${LOG_FILE}"}
${MAIL} -s """${EMAIL_SUBJECT}""" $EMAIL_FROM ${EMAIL_TO} < ${LOGFILE} cat ${LOGFILE} | ${MAIL} -s """${EMAIL_SUBJECT}""" $EMAIL_FROM ${EMAIL_TO}
echo -e "Email alert sent to ${EMAIL_TO} using ${MAIL}" >> ${LOGFILE} echo -e "Email alert sent to ${EMAIL_TO} using ${MAIL}" >> ${LOGFILE}
fi fi
fi fi
if [ ${ECHO} ]; then if [ ${ECHO} ]; then