351 lines
No EOL
11 KiB
Bash
351 lines
No EOL
11 KiB
Bash
#!/bin/bash
|
|
#
|
|
# Copyright (c) 2008-2009 Damon Timm.
|
|
#
|
|
# This program is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# This program is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
# ---------------------------------------------------------------------
|
|
#
|
|
# Version 3 - Jan 31 2009
|
|
# Incremental Encrypted Backups with Duplicity and Amazon S3
|
|
#
|
|
# This bash script was designed to automate and simplify the remote backup
|
|
# process using duplicity and Amazon S3. Hopefully, after the script is
|
|
# configured, you can easily backup, restore, verify and clean without
|
|
# having to remember lots of different command options.
|
|
#
|
|
# Furthermore, you can even automate the process of saving your script
|
|
# and the gpg key for your backups in a single password-protected file --
|
|
# this way, you know you have everything you need for a restore,
|
|
# in case your machine goes down.
|
|
#
|
|
# You can run the script from cron with no command-line options
|
|
# (all options set in the script itself); however, you can also run it
|
|
# outside of the cron with some variables for more control.
|
|
#
|
|
# --full: forces a full backup (instead of waiting specified number of days
|
|
# --verify: verifies the backup (no cleanup is run)
|
|
# --restore: restores the backup to the directory specified in the script
|
|
# --backup-this-script: let's you backup the script and secret key to the
|
|
# current working directory.
|
|
#
|
|
# See more info about the script online at:
|
|
# blog.damontimm.com/bash-script-incremental-encrypted-backups-duplicity-amazon-s3/
|
|
|
|
# TO DO:
|
|
# - allow command line restore options (specific files, etc)
|
|
# - allow command line cleanup options (# of days [30D] or full backups [2])
|
|
# - allow restore to specific path from the command line
|
|
|
|
# AMAZON S3 INFORMATION
|
|
export AWS_ACCESS_KEY_ID="<FOOBAR>"
|
|
export AWS_SECRET_ACCESS_KEY="<FOOBAR>"
|
|
|
|
# GPG PASSPHRASE & GPG KEY (Automatic/Cron Usage)
|
|
# If you aren't running this from a cron, comment this line out
|
|
# and duplicity should prompt you for your password.
|
|
# I put my GPG passphrase in a text file at
|
|
# ~/.gnupg/.gpg-passphrase and chmod it 0600.
|
|
export PASSPHRASE="<FOOBAR>"
|
|
GPG_KEY="<FOOBAR>"
|
|
|
|
# The ROOT of your backup (where you want the backup to start);
|
|
# This can be / or somwhere else -- I use /home/ because all the
|
|
# directories start with /home/ that I want to backup.
|
|
ROOT="/home/"
|
|
|
|
# BACKUP DESTINATION INFORMATION
|
|
# In my case, I use Amazon S3 use this - so I made up a unique
|
|
# bucket name (you don't have to have one created, it will do it
|
|
# for you. If you don't want to use Amazon S3, you can backup
|
|
# to a file or any of duplicity's supported outputs.
|
|
#
|
|
# NOTE: You do need to keep the "s3+http://<your location>/" format;
|
|
# even though duplicity supports "s3://<your location>/" this script
|
|
# needs to read the former.
|
|
#DEST="file:///home/damon/new-backup-test/"
|
|
DEST="s3+http://backup-bucket/backup-folder/"
|
|
|
|
# RESTORE FOLDER
|
|
# Being ready to restore is important to me, so I have this script
|
|
# setup to easily be able to restore a backup by adding the
|
|
# "--restore" flag. Indicate where you want the fili to restore to
|
|
# here so you're ready to go.
|
|
RESTORE="/home/damon/restore-backup-01"
|
|
|
|
# INCLUDE LIST OF DIRECTORIES
|
|
# Here is a list of directories to include; if you want to include
|
|
# everything that is in root, you could leave this list empty, I think.
|
|
#INCLIST=( "/home/*/Documents" \
|
|
# "/home/*/Projects" \
|
|
# "/home/*/logs" \
|
|
# "/home/www/mysql-backups" \
|
|
# )
|
|
|
|
INCLIST=( "/home/damon/Documents/Scripts/" ) # small dir for testing
|
|
|
|
# EXCLUDE LIST OF DIRECTORIES
|
|
# Even though I am being specific about what I want to include,
|
|
# there is still a lot of stuff I don't need.
|
|
EXCLIST=( "/home/*/Trash" \
|
|
"/home/*/Projects/Completed" \
|
|
"/**.DS_Store" "/**Icon?" "/**.AppleDouble" \
|
|
)
|
|
|
|
# FULL BACKUP & REMOVE OLDER THAN SETTINGS
|
|
# Because duplicity will continue to add to each backup as you go,
|
|
# it will eventually create a very large set of files. Also, incremental
|
|
# backups leave room for problems in the chain, so doing a "full"
|
|
# backup every so often isn't not a bad idea.
|
|
#
|
|
# I set the default to do a full backup every 14 days and to remove all
|
|
# all files over 31 days old. This should leave me at least two full
|
|
# backups available at any time, as well as a month's worth of incremental
|
|
# data.
|
|
|
|
FULL_IF_OLDER_THAN="14D"
|
|
CLEAN_UP_TYPE="remove-older-than"
|
|
CLEAN_UP_VARIABLE="31D"
|
|
|
|
# If you would rather keep a certain (n) number of full backups (rather
|
|
# than removing the files based on their age), uncomment the following
|
|
# two lines and select the number of full backups you want to keep.
|
|
# CLEAN_UP_TYPE="remove-all-but-n-full"
|
|
# CLEAN_UP_VARIABLE="5"
|
|
|
|
# LOGFILE INFORMATION DIRECTORY
|
|
# Provide directory for logfile, ownership of logfile, and verbosity level.
|
|
# I run this script as root, but save the log files under my user name --
|
|
# just makes it easier for me to read them and delete them as needed.
|
|
|
|
# LOGDIR="/dev/null"
|
|
LOGDIR="/home/damon/logs/test2/"
|
|
LOG_FILE="duplicity-`date +%Y-%m-%d-%M`.txt"
|
|
LOG_FILE_OWNER="damon:damon"
|
|
VERBOSITY="-v3"
|
|
|
|
##############################################################
|
|
# Script Happens Below This Line - Shouldn't Require Editing #
|
|
##############################################################
|
|
LOGFILE="${LOGDIR}${LOG_FILE}"
|
|
DUPLICITY="$(which duplicity)"
|
|
S3CMD="$(which s3cmd)"
|
|
|
|
NO_S3CMD="WARNING: s3cmd is not installed, remote file \
|
|
size information unavailable."
|
|
NO_S3CMD_CFG="WARNING: s3cmd is not configured, run 's3cmd --configure' \
|
|
in order to retrieve remote file size information."
|
|
|
|
if [ ! -x "$DUPLICITY" ]; then
|
|
echo "ERROR: duplicity not installed, that's gotta happen first!" >&2
|
|
exit 1
|
|
elif [ `echo ${DEST} | cut -c 1,2` = "s3" ]; then
|
|
if [ ! -x "$S3CMD" ]; then
|
|
echo $NO_S3CMD; S3CMD_AVAIL=FALSE
|
|
elif [ ! -f "${HOME}/.s3cfg" ]; then
|
|
echo $NO_S3CMD_CFG; S3CMD_AVAIL=FALSE
|
|
else
|
|
S3CMD_AVAIL=TRUE
|
|
fi
|
|
fi
|
|
|
|
if [ ! -d ${LOGDIR} ]; then
|
|
echo "Attempting to create log directory ${LOGDIR} ..."
|
|
if ! mkdir -p ${LOGDIR}; then
|
|
echo "Log directory ${LOGDIR} could not be created by this user: ${USER}"
|
|
echo "Aborting..."
|
|
exit 1
|
|
else
|
|
echo "Directory ${LOGDIR} successfully created."
|
|
fi
|
|
elif [ ! -w ${LOGDIR} ]; then
|
|
echo "Log directory ${LOGDIR} is not writeable by this user: ${USER}"
|
|
echo "Aborting..."
|
|
exit 1
|
|
fi
|
|
|
|
get_source_file_size()
|
|
{
|
|
echo "---------[ Source File Size Information ]---------" >> ${LOGFILE}
|
|
|
|
for exclude in ${EXCLIST[@]}; do
|
|
DUEXCLIST="${DUEXCLIST}${exclude}\n"
|
|
done
|
|
|
|
for include in ${INCLIST[@]}
|
|
do
|
|
echo -e $DUEXCLIST | \
|
|
du -hs --exclude-from="-" ${include} | \
|
|
awk '{ print $2"\t"$1 }' \
|
|
>> ${LOGFILE}
|
|
done
|
|
echo >> ${LOGFILE}
|
|
}
|
|
|
|
get_remote_file_size()
|
|
{
|
|
echo "------[ Destination File Size Information ]------" >> ${LOGFILE}
|
|
if [ `echo ${DEST} | cut -c 1,2` = "fi" ]; then
|
|
TMPDEST=`echo ${DEST} | cut -c 6-`
|
|
SIZE=`du -hs ${TMPDEST} | awk '{print $1}'`
|
|
elif [ `echo ${DEST} | cut -c 1,2` = "s3" ] && [ -x "$S3CMD" ]; then
|
|
TMPDEST=$(echo ${DEST} | cut -c 11-)
|
|
SIZE=`s3cmd du -H s3://${TMPDEST} | awk '{print $1}'`
|
|
fi
|
|
echo "Current Remote Backup File Size: ${SIZE}" >> ${LOGFILE}
|
|
}
|
|
|
|
include_exclude()
|
|
{
|
|
for include in ${INCLIST[@]}
|
|
do
|
|
TMP=" --include="$include
|
|
INCLUDE=$INCLUDE$TMP
|
|
done
|
|
for exclude in ${EXCLIST[@]}
|
|
do
|
|
TMP=" --exclude "$exclude
|
|
EXCLUDE=$EXCLUDE$TMP
|
|
done
|
|
EXCLUDEROOT="--exclude=**"
|
|
}
|
|
|
|
duplicity_cleanup()
|
|
{
|
|
echo "-----------[ Duplicity Cleanup ]-----------" >> ${LOGFILE}
|
|
${DUPLICITY} ${CLEAN_UP_TYPE} ${CLEAN_UP_VARIABLE} --force \
|
|
--encrypt-key=${GPG_KEY} \
|
|
--sign-key=${GPG_KEY} \
|
|
${DEST} >> ${LOGFILE}
|
|
echo >> ${LOGFILE}
|
|
}
|
|
|
|
duplicity_backup()
|
|
{
|
|
${DUPLICITY} ${OPTION} ${FULL_IF_OLDER_THAN} \
|
|
${VERBOSITY} \
|
|
--encrypt-key=${GPG_KEY} \
|
|
--sign-key=${GPG_KEY} \
|
|
${EXCLUDE} \
|
|
${INCLUDE} \
|
|
${EXCLUDEROOT} \
|
|
${ROOT} ${DEST} \
|
|
>> ${LOGFILE}
|
|
|
|
|
|
}
|
|
|
|
get_file_sizes()
|
|
{
|
|
get_source_file_size
|
|
get_remote_file_size
|
|
|
|
sed -i '/-------------------------------------------------/d' ${LOGFILE}
|
|
chown ${LOG_FILE_OWNER} ${LOGFILE}
|
|
}
|
|
|
|
backup_this_script()
|
|
{
|
|
if [ `echo ${0} | cut -c 1` = "." ]; then
|
|
SCRIPTFILE=$(echo ${0} | cut -c 2-)
|
|
SCRIPTPATH=$(pwd)${SCRIPTFILE}
|
|
else
|
|
SCRIPTPATH=$(which ${0})
|
|
fi
|
|
TMPDIR=DT-S3-Backup-`date +%Y-%m-%d`
|
|
TMPFILENAME=${TMPDIR}.tar.gpg
|
|
|
|
echo "You are backing up: "
|
|
echo " 1. ${SCRIPTPATH}"
|
|
echo " 2. GPG Secret Key: $GPG_KEY"
|
|
echo "Backup will be saved: `pwd`/${TMPFILENAME}"
|
|
echo
|
|
echo ">> Are you sure you want to do that ('yes' to continue)?"
|
|
read ANSWER
|
|
if [ "$ANSWER" != "yes" ]; then
|
|
echo "You said << ${ANSWER} >> so I am exiting now."
|
|
exit 1
|
|
fi
|
|
|
|
mkdir -p ${TMPDIR}
|
|
cp $SCRIPTPATH ${TMPDIR}/
|
|
gpg -a --export-secret-keys ${GPG_KEY} > ${TMPDIR}/s3-secret.key.txt
|
|
echo
|
|
echo "Encrypting tarball, choose a password you'll remember..."
|
|
tar c ${TMPDIR} | gpg -aco ${TMPFILENAME}
|
|
rm -Rf ${TMPDIR}
|
|
echo
|
|
echo ">> To restore files, run the following (remember your password!)"
|
|
echo "gpg -d ${TMPFILENAME} | tar x"
|
|
}
|
|
|
|
if [ "$1" = "--backup-this-script" ]; then
|
|
backup_this_script
|
|
exit
|
|
elif [ "$1" = "--full" ]; then
|
|
OPTION="full"
|
|
FULL_IF_OLDER_THAN=
|
|
include_exclude
|
|
duplicity_backup
|
|
duplicity_cleanup
|
|
get_file_sizes
|
|
|
|
elif [ "$1" = "--verify" ]; then
|
|
OLDROOT=${ROOT}
|
|
ROOT=${DEST}
|
|
DEST=${OLDROOT}
|
|
OPTION="verify"
|
|
FULL_IF_OLDER_THAN=
|
|
|
|
echo "-------[ Verifying Source & Destination ]-------" >> ${LOGFILE}
|
|
include_exclude
|
|
duplicity_backup
|
|
echo >> ${LOGFILE}
|
|
get_file_sizes
|
|
|
|
elif [ "$1" = "--restore" ]; then
|
|
ROOT=$DEST
|
|
DEST=$RESTORE
|
|
FULL_IF_OLDER_THAN=
|
|
OPTION=
|
|
|
|
if [ "$2" != "yes" ]; then
|
|
echo ">> You will restore to ${DEST}"
|
|
echo ">> You can override this question by executing '--verify yes' next time"
|
|
echo "Are you sure you want to do that ('yes' to continue)?"
|
|
read ANSWER
|
|
if [ "$ANSWER" != "yes" ]; then
|
|
echo "You said << ${ANSWER} >> so I am exiting now."
|
|
exit 1
|
|
fi
|
|
echo "Restoring now ..."
|
|
fi
|
|
duplicity_backup
|
|
|
|
elif [ "$1" = "--test" ]; then
|
|
echo "This was a non-duplicity scripting test: check logfile for file sizes."
|
|
get_file_sizes
|
|
else
|
|
OPTION="--full-if-older-than"
|
|
|
|
include_exclude
|
|
duplicity_backup
|
|
duplicity_cleanup
|
|
get_file_sizes
|
|
|
|
fi
|
|
|
|
unset AWS_ACCESS_KEY_ID
|
|
unset AWS_SECRET_ACCESS_KEY
|
|
unset PASSPHRASE |