diff --git a/README.md b/README.md index 71ec16f..d17927f 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,14 @@ # bgrestore -bgrestore-auto.sh helps you automatically restore the last successful backup from bgbackup to another MariaDB server. This is very useful for verifying backups, refreshing development environments, etc. - -bgrestore connects to the backup_history table in mdbutil to gather information about the last successful full backup such as whether it's encrypted, compressed, etc. This information is then used to restore on the designated MariaDB server. +bgrestore.sh helps you automatically restore the last successful full backup from bgbackup to another MariaDB server. This is very useful for verifying backups, refreshing development environments, etc. +bgrestore connects to the backup_history table in mdbutil to find the last successful full backup for a given host. It then delegates the actual restore (decrypt, decompress, prepare, move-back) to [fgrestore](../mysql-mariadb-physical-backup/files/fgrestore.sh), which needs to be installed alongside bgrestore on the restore host. bgrestore itself only keeps what's genuinely its own: the backup_history lookup, logging/mail, and stopping/starting the MariaDB service around the restore. ## How to... -This all assumes the backup was taken with bgbackup (https://github.com/bstillman/bgbackup). This is necessary for the script to gather the required information. +This all assumes the backup was taken with bgbackup (https://github.com/bstillman/bgbackup). This is necessary for the script to gather the required information. -Copy bgrestore.cnf.dist to /etc/bgrestore.cnf and configure as needed (details below). +Copy bgrestore.cnf.dist to /etc/bgrestore.cnf and configure as needed (details below). To run restore tests against a multi-instance host, copy it to a distinct path per instance instead and select it with `bgrestore -c /path/to/that/instance's/bgrestore.cnf`. Currently the script assumes the location of the backup on the source and the destination is the same. Ex: if the backup is in /backups on the server backed up, it should also reside in /backups on the server to be restored. @@ -70,13 +69,13 @@ The hostname of the server from which the backup was taken. Ex. If backing up se The full path to the data directory on the MariaDB server to be restored. -### datadirowner +### restore_my_cnf_file -The owner of the data directory. Usually mysql. +The my.cnf fgrestore should use for the restore. Parses datadir, tmpdir, innodb log/doublewrite dirs, log-error/log-bin/relay-log paths, and the owning `user=` (fgrestore chowns restored files to whatever `user=` says there, default mysql, if it's not set). -### datadirgroup +### service_name -The group of the data directory. Usually mysql. +The systemd service name to stop/start around the restore. Defaults to `mariadb`; set per-instance (via a distinct config file selected with `-c`) to run restore tests against a multi-instance host. ### logpath diff --git a/bgrestore-auto.sh b/bgrestore-auto.sh deleted file mode 100644 index 319ba6d..0000000 --- a/bgrestore-auto.sh +++ /dev/null @@ -1,222 +0,0 @@ -#!/bin/bash - -# bgrestore - Automate the restore of backups taken with bgbackup script. Great for backup verification, development refreshes, etc. -# -# Authors: Ben Stillman -# License: GNU General Public License, version 3. -# Redistribution/Reuse of this code is permitted under the GNU v3 license. -# As an additional term ALL code must carry the original Author(s) credit in comment form. -# See LICENSE in this directory for the integral text. - - - -# Functions - -# Handle control-c -function sigint { - echo "User has canceled with control-c." - # 130 is the standard exit code for SIGINT - exit 130 -} - -# Mail function -function mail_log { - mail -s "$mailsubpre $HOSTNAME Restore $log_status $mdate" "$maillist" < "$logfile" -} - -# Logging function -function log_info() { - if [ "$verbose" == "no" ] ; then - printf "%s --> %s\n" "$(date +%Y-%m-%d-%T)" "$*" >>"$logfile" - else - printf "%s --> %s\n" "$(date +%Y-%m-%d-%T)" "$*" | tee -a "$logfile" - fi - if [ "$syslog" = yes ] ; then - logger -p local0.notice -t bgrestore "$*" - fi -} - -# Preflight checks -function preflight { - # find and source the config file - etccnf=$( find /etc -name bgrestore.cnf ) - scriptdir=$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) - if [ -e "$etccnf" ]; then - source "$etccnf" - elif [ -e "$scriptdir"/bgrestore.cnf ]; then - source "$scriptdir"/bgrestore.cnf - else - echo "Error: bgrestore.cnf configuration file not found" - echo "The configuration file must exist somewhere in /etc or" - echo "in the same directory where the script is located" - log_status=FAILED - exit 1 - fi - # set logfile - logfile=$logpath/bgrestore_$(date +%Y-%m-%d-%T).log # logfile - # Check for xtrabackup - if command -v innobackupex >/dev/null; then - innobackupex=$(command -v innobackupex) - else - log_info "xtrabackup/innobackupex does not appear to be installed. Please install and try again." - log_status=FAILED - mail_log - exit 1 - fi - if [ "$datadir" == '' ] ; then - log_info "Datadir location not set correctly." - log_status=FAILED - mail_log - exit 1 - fi - # verify the backup prep directory exists - if [ ! -d "$preppath" ] - then - log_info "Error: $preppath directory not found" - log_info "The configured directory for backup prep does not exist." - log_status=FAILED - mail_log - exit 1 - fi - # verify user running script has permissions needed to write to backup prep directory - if [ ! -w "$preppath" ]; then - log_info "Error: $preppath directory is not writable." - log_info "Verify the user running this script has write access to the configured backup prep directory." - log_status=FAILED - mail_log - exit 1 - fi -} - -# Function to build mysql command -function mysqlhistcreate { - mysql=$(command -v mysql) - mysqlhistcommand="$mysqlcommand" - mysqlhistcommand=$mysqlhistcommand" -u $backuphistuser" - mysqlhistcommand=$mysqlhistcommand" -p$backuphistpass" - mysqlhistcommand=$mysqlhistcommand" -h $backuphisthost" - [ -n "$backuphistport" ] && mysqlhistcommand=$mysqlhistcommand" -P $backuphistport" - mysqlhistcommand=$mysqlhistcommand" -Bse " -} - -# Function to build mysql command -function mysqlshutdowncreate { - mysqlshutdowncommand="$mysqlcommand" - mysqlshutdowncommand=$mysqlshutdowncommand" -u $restoreuser" - mysqlshutdowncommand=$mysqlshutdowncommand" -p$restorepass" - mysqlshutdowncommand=$mysqlshutdowncommand" -h $restorehost" - [ -n "$restoreport" ] && mysqlshutdowncommand=$mysqlshutdowncommand" -P $restoreport" - mysqlshutdowncommand=$mysqlshutdowncommand" -Bse " -} - -# Function to get directory and other info from last full backup -function lastfullinfo { - mysqlhistcreate - lastfulluuid=$($mysqlhistcommand "select uuid from $backuphistschema.backup_history where butype = 'Full' and status = 'SUCCEEDED' and hostname = '$backuphost' and deleted_at = 0 order by end_time desc limit 1") - lastfullbulocation=$($mysqlhistcommand "select bulocation from $backuphistschema.backup_history where uuid = '$lastfulluuid' ") - if [ "$lastfullbulocation" == '' ] ; then - log_info "Backup location not set successfully." - log_status=FAILED - mail_log - exit 2 - fi - if [ ! -d "$lastfullbulocation" ] ; then - log_info "Error: $lastfullbulocation directory not found" - log_info "The directory for the last full backup cannot be found on this server." - log_status=FAILED - mail_log - exit 1 - fi - lastfullbktype=$($mysqlhistcommand "select bktype from $backuphistschema.backup_history where uuid = '$lastfulluuid' ") - if [ "$lastfullbktype" != "directory" ] ; then - log_info "$lastfullbktype not yet supported." - log_status=FAILED - mail_log - exit 2 - fi - lastfullcompressed=$($mysqlhistcommand "select compressed from $backuphistschema.backup_history where uuid = '$lastfulluuid' ") - lastfullencrypted=$($mysqlhistcommand "select encrypted from $backuphistschema.backup_history where uuid = '$lastfulluuid' ") - if [ "$lastfullencrypted" == "yes" ] ; then - lastfullcryptkey=$($mysqlhistcommand "select cryptkey from $backuphistschema.backup_history where uuid = '$lastfulluuid' ") - fi - log_info "Last full backup to restore: $lastfullbulocation " -} - -# Function to prepare backup for restore -function prepit { - cp -R "$lastfullbulocation" "$preppath"/ - buname=$(basename "$lastfullbulocation") - bufullpath="$preppath"/"$buname" - if [ "$lastfullencrypted" == "yes" ] ; then - log_info "Backup is encrypted." - $innocommand --decrypt=AES256 --encrypt-key="$(cat "$lastfullcryptkey")" --parallel="$threads" "$bufullpath" - for i in `find $bufullpath -iname "*\.xbcrypt"`; do rm -f $i; done - log_info "Backup now decrypted." - fi - if [ "$lastfullcompressed" == "yes" ] ; then - log_info "Backup is compressed." - $innocommand --decompress --parallel="$threads" "$bufullpath" - for i in `find $bufullpath -iname "*\.qp"`; do rm -f $i; done - log_info "Backup is now decompressed." - fi - $innocommand --apply-log "$bufullpath" - log_info "Backup has been prepared for restore." -} - -# Function to restore -function restoreit { - log_info "Shutting down MariaDB to restore. " - mysqlshutdowncreate -# $mysqlshutdowncommand "shutdown" - service mysql stop - log_info "Deleting the data directory." - rm -Rf "${datadir:?}"/* - log_info "Copying the backup to the data directory." - $innocommand --copy-back "$bufullpath" - log_info "Fixing privileges." - chown -R "$datadirowner":"$datadirgroup" "$datadir" - log_info "Starting MariaDB." - service mysql start - startstatus=$? - if [ "$startstatus" -eq 0 ] ; then - log_status=SUCCEEDED - log_info "MariaDB succussfully restored and restarted." - else - log_status=FAILED - log_info "Something went wrong. MariaDB did not start. Check error log." - exit 1 - fi -} - -# Cleanup the decompressed/decrypted backup copy -function cleanup { - if [ "$log_status" == "SUCCEEDED" ] ; then - log_info "Cleaning up." - rm -Rf "${bufullpath:?}" - log_info "Complete." - fi -} - - - -##### Begin script - -# we trap control-c -trap sigint INT - -# Set some specific variables -starttime=$(date +"%Y-%m-%d %H:%M:%S") -mdate=$(date +%m/%d/%y) # Date for mail subject. Not in function so set at script start time, not when backup is finished. -mysqlcommand=$(command -v mysql) -innocommand=$(command -v innobackupex) - -# do the work -preflight -lastfullinfo -prepit -restoreit -cleanup - -# email the log -mail_log - diff --git a/bgrestore.cnf.dist b/bgrestore.cnf.dist index 24d644c..17ed554 100644 --- a/bgrestore.cnf.dist +++ b/bgrestore.cnf.dist @@ -2,12 +2,21 @@ # bgrestore configuration file +# Path to the my.cnf fgrestore should use for the restore (parses datadir, tmpdir, +# innodb log/doublewrite dirs, log-error/log-bin/relay-log paths, and the owning +# 'user=' for chown). Same default fgrestore itself falls back to. +restore_my_cnf_file=/etc/my.cnf + +# systemd service name to stop/start around the restore. Needed to target a specific +# instance when running restore tests on a multi-instance host; select which +# bgrestore.cnf (and thus which service_name) via '-c|--config'. +service_name=mariadb # MariaDB host for restore -restorehost=127.0.0.1 +restorehost=localhost # MariaDB port for restore -restoreport=40001 +restoreport=3306 # MariaDB user for restore restoreuser=bgbackup @@ -19,10 +28,10 @@ restorepass=password preppath=/backup/tmp # MariaDB host for backup history -backuphisthost=127.0.0.1 +backuphisthost=localhost # MariaDB port for backup history -backuphistport=40002 +backuphistport=3306 # MariaDB user for backup history backuphistuser=bgbackup @@ -36,18 +45,18 @@ backuphistschema=mdbutil # MariaDB host which backup should be from backuphost=hostname_from_backup_history +# Skip copy step, place backup in preppath after creating it. This will also make bgrestore delete the contents of the folder instead of the folder itself (after succeeding) +skipcopy=no + # Datadir datadir=/var/lib/mysql -# Datadir owner -datadirowner=mysql - -# Datadir group -datadirgroup=mysql - # Path to keep logs logpath=/var/log +# Number of logs to keep +keeplognum=1000 + # Log to syslog syslog=no diff --git a/bgrestore.sh b/bgrestore.sh new file mode 100755 index 0000000..21af1a5 --- /dev/null +++ b/bgrestore.sh @@ -0,0 +1,274 @@ +#!/bin/bash + +# bgrestore - Automate the restore of backups taken with bgbackup script. Great for backup verification, development refreshes, etc. +# +# Authors: Ben Stillman , Michaƫl de Groot +# License: GNU General Public License, version 3. +# Redistribution/Reuse of this code is permitted under the GNU v3 license. +# As an additional term ALL code must carry the original Author(s) credit in comment form. +# See LICENSE in this directory for the integral text. + + + +# Functions + +# Handle control-c +function sigint { + echo "User has canceled with control-c." + # 130 is the standard exit code for SIGINT + exit 130 +} + +# Mail function +function mail_log { + mail -s "$mailsubpre $HOSTNAME Restore $log_status $mdate" "$maillist" < "$logfile" +} + +# Logging function +function log_info() { + if [ "$verbose" == "no" ] ; then + printf "%s --> %s\n" "$(date +%Y-%m-%d-%T)" "$*" >>"$logfile" + else + printf "%s --> %s\n" "$(date +%Y-%m-%d-%T)" "$*" | tee -a "$logfile" + fi + if [ "$syslog" = yes ] ; then + logger -p local0.notice -t bgrestore "$*" + fi +} + +# Error function +function log_error() { + if [ "$syslog" = yes ] ; then + logger -p local0.notice -t bgrestore "$*" + fi + printf "%s --> %s\n" "$(date +%Y-%m-%d-%T)" "$*" >>"$logfile" + printf "%s --> %s\n" "$(date +%Y-%m-%d-%T)" "$*" 1>&2 + exit 1 +} + + +# Preflight checks +function preflight { + # source the config file (path resolved by CLI arg parsing / default, see "Begin script") + if [ -e "$etccnf" ]; then + source "$etccnf" + elif [ -e "$scriptdir"/bgrestore.cnf ]; then + source "$scriptdir"/bgrestore.cnf + else + echo "Error: bgrestore.cnf configuration file not found" + echo "The configuration file must exist somewhere in /etc or" + echo "in the same directory where the script is located" + log_status=FAILED + exit 1 + fi + # set logfile + logfile=$logpath/bgrestore_$(date +%Y-%m-%d-%T).log # logfile + + if [ "$datadir" == '' ] ; then + log_info "Datadir location not set correctly." + log_status=FAILED + mail_log + exit 1 + fi + # verify the backup prep directory exists + if [ ! -d "$preppath" ] + then + log_info "Error: $preppath directory not found" + log_info "The configured directory for backup prep does not exist." + log_status=FAILED + mail_log + exit 1 + fi + # verify user running script has permissions needed to write to backup prep directory + if [ ! -w "$preppath" ]; then + log_info "Error: $preppath directory is not writable." + log_info "Verify the user running this script has write access to the configured backup prep directory." + log_status=FAILED + mail_log + exit 1 + fi +} + +# Function to build mysql command +function mysqlhistcreate { + mysql=$(command -v mysql) + mysqlhistcommand="$mysqlcommand" + mysqlhistcommand=$mysqlhistcommand" -u $backuphistuser" + mysqlhistcommand=$mysqlhistcommand" -p$backuphistpass" + mysqlhistcommand=$mysqlhistcommand" -h $backuphisthost" + [ -n "$backuphistport" ] && mysqlhistcommand=$mysqlhistcommand" -P $backuphistport" + mysqlhistcommand=$mysqlhistcommand" -Bse " +} + +# Function to build mysql command +function mysqlshutdowncreate { + mysqlshutdowncommand="$mysqlcommand" + mysqlshutdowncommand=$mysqlshutdowncommand" -u $restoreuser" + mysqlshutdowncommand=$mysqlshutdowncommand" -p$restorepass" + mysqlshutdowncommand=$mysqlshutdowncommand" -h $restorehost" + [ -n "$restoreport" ] && mysqlshutdowncommand=$mysqlshutdowncommand" -P $restoreport" + mysqlshutdowncommand=$mysqlshutdowncommand" -Bse " +} + +# Function to get directory and other info from the last backup (any type -- fgrestore +# is chain-aware, so a Differential/Incremental gets trickled back to its Full automatically) +function lastbackupinfo { + mysqlhistcreate + lastbuuuid=$($mysqlhistcommand "select uuid from $backuphistschema.backup_history where status = 'SUCCEEDED' and hostname = '$backuphost' and (deleted_at IS NULL OR deleted_at = 0) order by end_time desc limit 1") + lastbulocation=$($mysqlhistcommand "select bulocation from $backuphistschema.backup_history where uuid = '$lastbuuuid' ") + if [ "$lastbulocation" == '' ] ; then + log_info "Backup location not set successfully." + log_status=FAILED + mail_log + exit 2 + fi + if [ ! -d "$lastbulocation" ] && [ "$skipcopy" != "yes" ] ; then + + log_info "Error: $lastbulocation directory not found" + log_info "The directory for the last backup cannot be found on this server." + log_status=FAILED + mail_log + exit 1 + fi + + log_info "Last backup to restore: $lastbulocation " +} + +# Cleanup the decompressed/decrypted backup copy +# Regardless of skipcopy, fgrestore always ends up with the prepared backup flattened +# directly into $preppath (skipcopy=yes: prepared in place there via '-I'; skipcopy=no: +# fgrestore itself copies into it via '-D') -- so cleanup is now the same either way. +# Also sweeps fgrestore's chain-staging dirs ('.inc.*') -- bgrestore now +# restores the latest backup of any type, so a Differential/Incremental chain of more +# than one member does create these; -M's --move-back only moves $preppath itself back, +# not the per-member staging dirs. +function cleanup { + if [ "$log_status" == "SUCCEEDED" ] ; then + log_info "Cleaning up." + rm -Rf "${preppath:?}"/* + rm -Rf "${preppath:?}".inc.* + log_info "Complete." + fi +} + +# Function to cleanup logs +function log_cleanup { + if [ $log_status = "SUCCEEDED" ]; then + delloglist=$(ls -tp "$logpath" | grep bgrestore | tail -n +$((keeplognum+=1))) + for logtodelete in $delloglist; do + rm -f "$logpath"/"$logtodelete" + log_info "Deleted log file $logpath/$logtodelete" + done + else + log_info "Restore failed. Not deleting any log files at this time." + fi +} + +##### Begin script + +# we trap control-c +trap sigint INT + +scriptdir=$( cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +etccnf="/etc/bgrestore.cnf" + +# Function to display usage +usage() { + echo "Usage: $0 [-c config_file | --config config_file]" + exit 1 +} + +# Parse command line arguments +while [[ "$#" -gt 0 ]]; do + case "$1" in + -c|--config) + if [[ -n "${2:-}" ]]; then + etccnf="$2" + shift + else + echo "Error: --config requires a non-empty option argument." + usage + fi + ;; + *) + echo "Error: Unknown option $1" + usage + ;; + esac + shift +done + +# Set some specific variables +starttime=$(date +"%Y-%m-%d %H:%M:%S") +mdate=$(date +%m/%d/%y) # Date for mail subject. Not in function so set at script start time, not when backup is finished. +mysqlcommand=$(command -v mysql) + +# do the work +preflight + +# Check that we are not already running. Scoped per service_name so separately +# configured instances on the same restore host (multi-instance restore testing) can +# run concurrently, while two runs against the same instance still can't overlap. +lockfile=/tmp/bgrestore +[ -n "$service_name" ] && lockfile=$lockfile"-$service_name" +lockfile=$lockfile".lock" + +if [ -f $lockfile ] +then + log_error "Another instance of $lockfile is already running. Exiting." +fi +trap 'rm -f $lockfile' 0 +touch $lockfile + +lastbackupinfo + +log_info "Shutting down MariaDB to restore." +mysqlshutdowncreate +$mysqlshutdowncommand "shutdown" + +# Decrypt/decompress/prepare/move-back are all handled by fgrestore, chain-aware +# (Full/Differential/Incremental). '-r' always removes compressed originals after +# decompression. skipcopy=yes means copy-last-backup.sh already rsynced the backup +# straight into preppath, so '-I' (in-place) prepares it there directly -- a second +# copy would double disk usage. Otherwise fgrestore copies from lastbulocation +# into preppath itself via '-D'. +if [ "$skipcopy" == "yes" ]; then + fgrestore -S "$preppath" -C "$restore_my_cnf_file" -M -N -r -I \ + $( [ "$run_restorecon" == "yes" ] && echo -R ) >> "$logfile" 2>&1 +else + fgrestore -S "$lastbulocation" -D "$preppath" -C "$restore_my_cnf_file" -M -N -r \ + $( [ "$run_restorecon" == "yes" ] && echo -R ) >> "$logfile" 2>&1 +fi +fgrestorestatus=$? +if [ "$fgrestorestatus" -eq 0 ] ; then + log_info "fgrestore completed successfully." +else + log_status=FAILED + log_info "Something went wrong. fgrestore failed. See $logfile for details." + mail_log + exit 1 +fi + +log_info "Fixing unfinished transactions. MDEV-6660 workaround." +sudo -u mysql mysqld --tc-heuristic-recover=ROLLBACK + +log_info "Starting MariaDB." +systemctl start "$service_name" +startstatus=$? +if [ "$startstatus" -eq 0 ] ; then + log_status=SUCCEEDED + log_info "MariaDB succussfully restored and restarted." +else + log_status=FAILED + log_info "Something went wrong. MariaDB did not start. Check error log." + mail_log + exit 1 +fi + +cleanup + +# email the log +mail_log + +# clean old log ifles +log_cleanup