#!/bin/bash SHELL=/bin/bash # for Debian /etc/crontab which uses /bin/sh for SHELL PATH=/usr/sbin:/usr/bin:/sbin:/bin # Backup to a usb drive # # This program was written to back up a Linux system to a (possibly unmounted) # usb drive but it can be used to backup any directory to any drive or directory. # # Usage: backup_to_usb [ --before=prog ] [ --after=prog ] [ [ rsync option ] ... source_directory ] # If source_directory is not defined, it is set to / ($DEFAULT_SOURCE_DIR setting). # # Version 0.97 # Copyright 2012-2019, Mack Pexton (mack@mackpexton.com) # License https://www.opensource.org/licenses/mit-license.php # # Configuration # # Extra program log messages are generated by defining the DEBUG variable. DEBUG=${DEBUG:-""} # use environment variable setting if defined # Default directory to backup if not specified on command line. DEFAULT_SOURCE_DIR="/" # In order for a drive (or directory) to be used as a backup drive, it must # have a top level directory with the name defined below by BACKUP_DIR_NAME. BACKUP_DIR_NAME="backup" # The backup id file is an optional file that is stored in the BACKUP_DIR_NAME # directory. Its contents are copied to the output before the backup begins. # This file is intended to identify the disk drive being used to backup. BACKUP_ID_NAME="backup-id" # The following mount points are first checked for drives (or directories) that are # already mounted and have the backup directory. Enter only one directory name per line. MOUNTED_DIRS=" /mnt/usb /media/disk " # If none of the above mount points has a backup directory, check the following # (usb) devices and mount them if they are not already mounted. File globs can # be used to specify the disk drives. The following disk drives are first checked # with drives that are actually attached to computer as listed in /proc/partitions. DEVICES=" /dev/sda[1-4] /dev/sdb[1-4] /dev/sdc[1-4] /dev/sdd[1-4] /dev/sde[1-4] /dev/sdf[1-4] /dev/sdg[1-4] " # Define the automount settings used to define a mount point if automatically mounting a drive. AUTOMOUNT_PATH="/mnt" # default place where drives are to be mounted AUTOMOUNT_NAME="backup-$(date +'%Y%m%d_%H%M')" # directory name for automount (e.g. backup-20100116_0410) AUTOMOUNT_DIR="$AUTOMOUNT_PATH/$AUTOMOUNT_NAME" # directory path where drive is to be mounted AUTOMOUNT_REMOVE_DIR_WHEN_DONE=yes # set to 'yes' to remove mount directory after drive unmounted # Use the following rsync program options. RSYNC="$(which rsync)" # Command line arguments can add to or override the following settings. RSYNC_OPT="-Sa --numeric-ids" # Following is incompatable with hard-links incremental backup #DELETE="--delete-before" # Delete files in backup that are not in source before transfer DELETE="--delete" # Delete files in backup that are not in source HARD_LINKS="--hard-links" # Helpful for backing up rsnapshot backups #VERBOSE="-v" VERBOSE="--info=STATS" EXCLUDES=" --exclude=/dev/ --exclude=/lost+found/ --exclude=/media/ --exclude=/mnt/ --exclude=/proc/ --exclude=/sys/ --exclude=/tmp/ --exclude=/var/cache/ --exclude=/var/lock/ --exclude=/var/run/ --exclude=/var/tmp/ " # This program generates two log files: a program log and a backup log. The program log # monitors the program actions and their results and the backup log records the files # backed up or missed. The program log is written to the terminal if this program is # invoked from the command line, otherwise the Linux logger program is used to write # messages to the system log files. if [ -t 0 ] # if program invoked from a terminal then # Write log message to terminal. PROGRAM_LOGGER="eval cat >/dev/tty" PROGRAM_ERROR_LOGGER="eval (echo -n 'Error: ' >/dev/tty; cat >/dev/tty)" else # Use Linux logger program. PROGRAM_LOGGER="logger -p syslog.notice -t $(basename $0)" PROGRAM_ERROR_LOGGER="logger -p syslog.err -t $(basename $0)" fi # The backup log is written to the backup directory. For example, if the backup directory # is /mnt/usb_drive/backup/computer_name then the backup log is written to the file # /mnt/usb_drive/backup/computer_name.log. If the KEEP_ALL_BACKUP_LOGS variable equals "yes" # then the backup log file names will also contain a timestamp of the backup so that future # backups won't overwrite the log file. In other words, the backup log would be written to a # file like /mnt/usb_drive/backup/computer_name-20100120_0400.log. KEEP_ALL_BACKUP_LOGS=yes # set to 'yes' to avoid overwriting previous log files # The backup log can optionally be emailed to the following address(es) when backup is done. # Do not define an address if you do not want the backup log to be emailed. TO_EMAIL_ADDRESS="root@$(hostname --fqdn)" FROM_EMAIL_ADDRESS="$(id -un)@$(hostname --fqdn)" SENDMAIL="/usr/sbin/sendmail -oi -i -t" # sendmail program with options to send immediately # External drives can be formatted any way this system recognizes. # FAT -- does not handle very large files, no ownership or file privileges are kept # FAT32 -- no file ownership or privileges are kept # NTFS -- after creating backup directory on drive, make sure it is writable. # -- had problems backing up jpg files with file names using utf-8 foreign characters. (2013) # -- might have to install ntfs-3g to create backup directory: apt-get install ntfs-3g # -- seems to take inordinately long to write to disk (13hrs for 500GB) # ext2 -- preferred Linux format. The following formats a drive from the command line. # # To find the drive associated to the external usb drive, do: # # fdisk -l # To format the drive (assuming the external usb drive is /dev/sdc1): # # mkfs.ext2 /dev/sdc1 # To make the drive a backup drive, create the backup directory on it: # # mount /dev/sdc1 /mnt # # mkdir /mnt/backup # assuming BACKUP_DIR_NAME=backup above # # umount /dev/sdc1 # finish # Consider installing a bootable live distribution on the usb drive first. That would allow the # computer to be booted off the usb drive and have ability to repair and restore the entire system. # Sometimes when auto-mounting disk drives the file system type cannot be automatically determined. # This can happen when adding a USB card to the computer. If you know what kind of file system # is always used to backup you can specify it in FSTYPE. If FSTYPE is blank or not defined, the # system will try to automatically determine which kind of file system to mount. FSTYPE="" # End of configuration # # Program Arguments # # The first command line arguments are examined as program arguments to backup_to_usb. The remaining # arguments are passwd to the rsync program doing the backup. A lone -- can be used to terminate # arguments to backup_to_usb. while true do case "$1" in --before=*) BEFORE_PROG="${1#--before=}"; shift ;; --after=*) AFTER_PROG="${1#--after=}"; shift ;; --) shift; break ;; ""|*) break ;; esac done # Determine source directory to backup. # Last argument can specify the source directory to backup. # Set SOURCE_DIR to the default only if last argument is not a backup source. if [ $# -gt 0 ] then eval source_dir=\$$# if [ "${source_dir:0:1}" = "-" ] then # Last argument is a program option beginning with - SOURCE_DIR="$DEFAULT_SOURCE_DIR" fi else SOURCE_DIR="$DEFAULT_SOURCE_DIR" fi # # Functions # lock_program() { PROGNAME=${PROGNAME:-$(basename $0)} LOCKFILE="/var/lock/$PROGNAME.pid" LOCKPID=$(cat "$LOCKFILE" 2>/dev/null) if [ "$LOCKPID" ] && ps $LOCKPID &>/dev/null then echo "$(date): $PROGNAME ($LOCKPID) is already running." 1>&2 exit 1 fi echo $$ > "$LOCKFILE" trap "rm -f '$LOCKFILE' 2>/dev/null" EXIT # automatic unlock } make_backup_dir_name() { # Assemble the full path name of target backup directory. local mount_dir="$1" local required_backup_dir_name="$BACKUP_DIR_NAME" # Add a directory named after the computer name. local computer_name=$(hostname --short) # Add suffix to the backup directory name to allow multiple rotating backups. local suffix # Uncomment one of the following to add a suffix. #suffix="$(date +'%w')" # add day of week (0,1,...) #suffix="-$(date +'%a')" # add day of week (Sun, Mon,...) #suffix="-$(date +'%A')" # add day of week (Sunday, Monday,...) # Construct backup directory name, trim possible trailing / from mount_dir. echo "${mount_dir%%/}/$required_backup_dir_name/$computer_name$suffix" } program_log() { echo -e "$@" | $PROGRAM_LOGGER; } program_error() { echo -e "$@" | $PROGRAM_ERROR_LOGGER; } program_debug() { [ "$DEBUG" ] || return; echo -e "$@" | $PROGRAM_LOGGER; } email_backup_log() { local log_file="$1" if [ ! "$TO_EMAIL_ADDRESS" ] then program_debug "No email address is configured to receive backup log." return 0 fi # Send email. ( echo "From: $FROM_EMAIL_ADDRESS" echo "To: $TO_EMAIL_ADDRESS" echo "Subject: $(basename $0) log for $(hostname --fqdn)" echo "Content-Type: text/plain" echo cat "$log_file" 2>/dev/null || echo "Cannot access the backup log file \"$log_file\" ($?)." ) | $SENDMAIL local status=$? if [ $status -eq 0 ] then program_debug "Emailed backup log for $(hostname --fqdn) to $TO_EMAIL_ADDRESS." return 0 else program_error "Failed to email backup log for $(hostname --fqdn) to $TO_EMAIL_ADDRESS ($status)." return $status fi } email_error() { local msg="$1" tmp=$(mktemp $(basename $0)-ERROR.XXXXXX) trap "rm $tmp" EXIT echo "$msg" > "$tmp" email_backup_log "$tmp" } is_boot_device() { # Return true if given device (/dev/sda1) is the boot drive. [ "$bootdrive" ] || bootdrive=$(fdisk -l 2>/dev/null | sed -n -e '/^\(\/dev\/[^ ]*\) *\*.*/{s//\1/p;q}') # look for * in boot column of list local dev="$1" [ "$1" = "$bootdrive" ] && return 0 # yes return 1 # no } is_usb_device() { # Return true if given device (e.g. /dev/sdc1) is a usb device. local dev="$1" find /dev/disk | grep usb | while read f do ( cd $(dirname "$f") || continue realpath $(readlink "$f") ) done | grep -s "^$dev\$" &>/dev/null } is_backup_drive() { # Return true if given mount point has a backup directory defined in it. local mount_dir="$1" [ "$mount_dir" ] || return 1 # no local backup_dir="${mount_dir%%/}/$BACKUP_DIR_NAME" [ -d "$backup_dir" ] || return 1 # no # Check to ensure backup directory is not also a mounted drive. backup_mount=$(mount | perl -ne 's/.* on //; s/ type .*//; print if m|^'"$backup_dir"'\b|;') [ "$backup_mount" ] && { program_debug "Backup directory $backup_dir is also a mount point so cannot use $mount_dir as a backup drive." return 1 # no } return 0 # yes } attached_partitions() { # Remove top header lines and strip first fields and prepend /dev. sed -e '1,2d' -e 's/.* //' -e 's|^|/dev/|' < /proc/partitions } mounted_backup_dir() { # Search configured mount directories for a backup directory. check_mounted_drives() { # helper function to return correct exit status local mount_dir while read mount_dir # read one line (one directory name per line) do [ "$mount_dir" ] || continue if is_backup_drive "$mount_dir" then program_debug "Successfully found an already mounted backup drive or directory at $mount_dir." echo "$mount_dir" return 0 # success fi done program_debug "Did not find a configured directory or a drive that's already mounted with a directory named \"$BACKUP_DIR_NAME\"." return 1 # fail } if [ "$MOUNTED_DIRS" ] then echo "$MOUNTED_DIRS" | check_mounted_drives else program_debug "No directories are configured to check first for a \"$BACKUP_DIR_NAME\" directory." fi } premounted_disk_backup_dir() { # Search configured disk drives that are already mounted for a backup directory. local device dir for device in $DEVICES do [ "$device" ] || continue mount_dir="$(mount | grep "^$device " | awk '{print $3}')" if is_backup_drive "$mount_dir" then program_debug "Successfully found a premounted backup drive $device at $mount_dir." echo $mount_dir return 0 # success fi done program_debug "Did not find a premounted backup drive with a directory named \"$BACKUP_DIR_NAME\"." return 1 # fail } automounted_disk_backup_dir() { # Search attached drives that need to be mounted first for a backup directory. local partitions device dev # Define directory where drive is to be mounted. [ -d "$AUTOMOUNT_DIR" ] || mkdir -p "$AUTOMOUNT_DIR" # Define options to use for mount command. mount_opt='' [ "$FSTYPE" ] && mount_opt="-t $FSTYPE" for device in $DEVICES do [ "$device" ] || continue # Check if device is in list of all disk partitions attached to the computer. dev=$(attached_partitions | grep "^$device\$") [ "$dev" ] || continue is_usb_device "$device" || continue # only check usb devices is_boot_device "$device" && continue # safety # Exclude drives already mounted. { mount | grep -q "^$device "; } && continue # Exclude drives that are part of LVM. if [ -x /sbin/pvs -o -x /usr/bin/pvs ]; then { pvs | grep -q "^[[:space:]]*$device[[:space:]]"; } && continue fi # Mount device. # Some USB divices need extra time during a mount to spin up and to auto-determine the disk # filesystem type so we attempt to mount the device twice to make sure. mount $mount_opt $dev "$AUTOMOUNT_DIR" 2>/dev/null || { sleep 60; mount $mount_opt $dev "$AUTOMOUNT_DIR"; } || { program_debug "Cannot mount $dev to $AUTOMOUNT_DIR."; continue; } if is_backup_drive "$AUTOMOUNT_DIR" then program_debug "Successfully automounted backup drive $device to $AUTOMOUNT_DIR." echo "$AUTOMOUNT_DIR" return 0 # success fi umount "$AUTOMOUNT_DIR" done [ "$AUTOMOUNT_REMOVE_DIR_WHEN_DONE" = 'yes' -a -d "$AUTOMOUNT_DIR" ] && rmdir "$AUTOMOUNT_DIR" program_debug "Did not find a backup drive that could be automounted which has a directory named \"$BACKUP_DIR_NAME\"." return 1 # fail } backup_to_dir() { # Last argument is backup target directory, preceded by source directory and other possible rsync options. eval local backup_dir=\$$# eval local source_dir=\$$(($# - 1)) if [ ! -d "$backup_dir" ] then program_error "Backup drive or directory \"$backup_dir\" cannot be found." return 1 fi # Output disk drive ID file. [ -r "$backup_dir/../$BACKUP_ID_NAME" ] && { cat "$backup_dir/../$BACKUP_ID_NAME" echo } [ "$BEFORE_PROG" ] && source "$BEFORE_PROG" # Record backup command in backup log. echo $RSYNC $RSYNC_OPT $VERBOSE $DELETE $HARD_LINKS $EXCLUDES "$@" # Do backup. $RSYNC $RSYNC_OPT $VERBOSE $DELETE $HARD_LINKS $EXCLUDES "$@" status=$? [ "$AFTER_PROG" ] && source "$AFTER_PROG" return $status } # # Main program. # lock_program program_log "Backup started: $(date)" mount_dir="$(mounted_backup_dir || premounted_disk_backup_dir || automounted_disk_backup_dir)" if [ "$mount_dir" ] then # Create backup directory if needed. backup_dir="$(make_backup_dir_name "$mount_dir")" [ ! -d "$backup_dir" ] && mkdir -p "$backup_dir" program_log "Backing up to: $backup_dir" # Create log file next to backup directory. [ "$KEEP_ALL_BACKUP_LOGS" = 'yes' ] && backup_log="$backup_dir-$(date +'%Y%m%d_%H%M').log" || backup_log="$backup_dir.log" [ ! -f "$backup_log" ] && > "$backup_log" program_log "Backup log is: $backup_log" # Divert all program output to backup log file. exec 3>&1 4>&2 # temporarily save stdout and stderr in fd 3 and 4 # Save log file in /tmp to reduce armature rattle on backup drive. tmp_log_file="/tmp/$(basename "$backup_log")" exec > "$tmp_log_file" 2>&1 # send all output to temp backup log # Backup files. echo "Backup started: $(date)"; echo TIMEFORMAT='Total time: %0lR' time { backup_to_dir --exclude "$tmp_log_file" --exclude "${mount_dir%%/}/" "$@" $SOURCE_DIR "$backup_dir" status=$? } if [ $status -eq 0 ] then touch "$backup_dir" # change modification time of backup directory to time of backup else program_error "Backup finished with an error status: $status. Check the backup log." fi echo; df -BG $mount_dir; echo # record disk usage in backup log echo "Backup finished: $(date)" # Restore program output (close backup_log and allow drive to be unmounted). exec 1>&3 2>&4 # restore stdout and stderr (close log file) exec 3>&- 4>&- # close fd 3 and 4 # Move temporary log file onto backup drive. mv "$tmp_log_file" "$(dirname "$backup_dir")" email_backup_log "$backup_log" # Record disk usage in program log. program_log "Backup device: $(df -BG $mount_dir | awk 'END { print $1 " used " $3 "(" $5 "), " $4 " available"; }')" # Clean up. if [ -d "$AUTOMOUNT_DIR" ] then cd sleep 60 # pause to allow device to complete cached actions umount "$AUTOMOUNT_DIR" && program_debug "Successfully unmounted automounted directory '$AUTOMOUNT_DIR'." || program_error "Cannot unmount automounted directory '$AUTOMOUNT_DIR'." if [ "$AUTOMOUNT_REMOVE_DIR_WHEN_DONE" = 'yes' ] then rmdir "$AUTOMOUNT_DIR" || program_error "Cannot remove automounted directory '$AUTOMOUNT_DIR'." fi fi else msg="Cannot find a backup drive with a directory named \"$BACKUP_DIR_NAME\" to backup to." program_error "$msg" email_error "$msg" status=1 # error fi program_log "Backup finished: $(date)" exit $status