initial commit of containerised MariaDB backup functionality

This commit is contained in:
Dave Lane 2022-06-09 15:26:20 +12:00
parent 794ca288a5
commit 715b32f38f
3 changed files with 462 additions and 0 deletions

View file

@ -0,0 +1,22 @@
#
# SQL dump backup directory
#
BU_DIR=/home/data/mastodon/backup
#
# Docker Compose details
#
# dir containing the docker-compose.yml
DC_DIR=/home/docker/[dirname]
DC_FILE=$DC_DIR/docker-compose.yml
DC_CONTAINER=mariadb
DC_DB_NAME_VAR=MOODLE_DB_NAME
DC_DB_USER_VAR=MOODLE_DB_USER
DC_DB_PASSWORD_VAR=MOODLE_DB_PASSWORD
# Command to dump the relevant database(s)
DUMP_CMD='mysqldump '
#
# Reporting
#
# email address to send reports to, and subject
EMAIL=webmaster@oerfoundation.org
EMAIL_SUBJ="Mastodon (on Open) MariaDB Backup Report"

View file

@ -0,0 +1,240 @@
#!/bin/bash
#
# This script dumps the PostgreSQL db in a Docker container
# managed by Docker Compose using the selected command within
# the database container and pipes the results into an
# appropriately named/dated file...
#
# It also manages rolling out older backups to avoid filling
# your storage...
#
# Default retention
BU_TO_KEEP_HOURLY=24
BU_TO_KEEP_DAILY=7
BU_TO_KEEP_WEEKLY=4
BU_TO_KEEP_MONTHLY=12
BU_TO_KEEP_YEARLY=7
#
BU_FROOT=alldbs
BU_FROOT_HOURLY=hourly
BU_FROOT_DAILY=daily
BU_FROOT_WEEKLY=weekly
BU_FROOT_MONTHLY=monthly
BU_FROOT_YEARLY=yearly
#
# this can be overridden at invocation
BU_CONF=default-docker-compose.conf
# # output for debugging...
VERBOSE=0
#
# Stuff that should be universal for this install...
# where we can find this app...
MAIN_DIR=/home/data/scripts/mariadbbackup-docker-compose
# determine today's date
DATE=`date '+%Y-%m-%d-%a'`
# determine today's date
TIME=`date '+%H-%M-%S'`
# temporary holding point for email
TMP_EMAIL=/tmp/tmp_mariadbbackup_email.$DATE_$TIME
# log file
LOG=/var/log/mariadbbackup.log
#
# Commands
# gzip command
GZIP=`which gzip`
# grep command
GREP=`which grep`
# email program
MAIL=`which mail`
# database dump utility
DC=`which docker-compose`
#DEF_ARGS="-C -d -O -x"
#
# pattern for "ls" command to build list of
# pruneable backup files...
# -1t = 1 column, ordered by time of last mod
PRUNEABLES_CMD="ls -1t"
# function to direct a message...
message() {
#
# a timestamp for logging purposes
TIMESTAMP=`date '+%Y-%m-%d %H:%M.%S'`
echo "$0: $TIMESTAMP $@" >> $LOG
if test -f $TMP_EMAIL ; then
echo "$0: $TIMESTAMP $@" >> $TMP_EMAIL
fi
verbose "$TIMESTAMP $@"
}
# create the temporary email file
create_tmp_email() {
touch $TMP_EMAIL
if test -f $TMP_EMAIL ; then
message "created temporary email $TMP_EMAIL"
else
message "failed to create temporary email $TMP_EMAIL"
fi
}
# send the contents of the temporary file to the
# designated report recipient
send_email_report() {
if test -f $TMP_EMAIL ; then
message "sending email report to $EMAIL"
$MAIL -s "$EMAIL_SUBJ" $EMAIL < $TMP_EMAIL
rm $TMP_EMAIL
if test -f $TMP_EMAIL ; then
message "failed to remove temporary email $TMP_EMAIL"
else
message "successfully removed temporary email $TMP_EMAIL"
fi
message "email report successfully sent"
fi
}
# insert a blank line into the log and on the console
insert_blank() {
echo "" >> $LOG
verbose ""
}
# function to direct a message...
verbose() {
if test $VERBOSE = 1 ; then
echo "$@"
fi
}
#
# delete old backups
delete_old() {
#
verbose "deleting old files based on $1"
# pattern to search for to build the list...
PATTERN="$BU_DIR/$1-*.*"
# build the list, with the suffix...
PRUNEABLES=`$PRUNEABLES_CMD $PATTERN`
if test "$?" -eq "0" ; then
message "pruning older files based on $PATTERN"
BU_TO_KEEP=$2
message "keeping last $BU_TO_KEEP backups"
#
# set counter
NUM=0
# go through the list of files and remove those we don't want
for PRUNEABLE in $PRUNEABLES
do
NUM=$(($NUM + 1))
if test $NUM -gt $BU_TO_KEEP ; then
message "deleting $PRUNEABLE"
rm $PRUNEABLE 2>&1 > /dev/null
else
message "keeping $PRUNEABLE"
fi
done
else
message "No files with $PATTERN to delete"
fi
}
#
#
do_backup() {
FILE=$1
DATABASE="$(get_value DC_DB_NAME_VAR $DC_FILE)"
USER="$(get_value DC_DB_USER_VAR $DC_FILE)"
PASSWORD="$(get_value DC_DB_PASSWORD_VAR $DC_FILE)"
verbose "debug - DB details: USER = $USER, PASSWORD = $PASSWORD, DB = $DATABASE"
VER=`$DC exec $DC_CONTAINER mysql --version`
verbose "MariaDB version $VER - echoing to backup: $FILE"
echo "--" > $FILE
echo "-- MariaDB Version: $VER" >> $FILE
echo "--" >> $FILE
echo "" >> $FILE
CMD="$DC exec $DC_CONTAINER mysqldump $DATABASE -u $USER -p$PASSWORD"
verbose "doing database dump: $CMD"
$CMD >> $FILE
}
# $1 variable to look for
# $2 file to look in
get_value() {
echo `grep -m 1 "${1}" $2 | cut -d : -f 2 | cut -d '#' -f 1 | tr -d '[:space:]'`
}
#
# cycle through the command line options
while test $# -ne 0 ; do
case $1 in
--config|-c)
shift # shift from the flag to the value
verbose "setting configuration directory to $1"
BU_CONF=$1
;;
--hourly|-h)
verbose "running hourly backup"
TASK=HOURLY
;;
--daily|-d)
verbose "running daily backup"
TASK=DAILY
;;
--weekly|-w)
verbose "running weekly backup"
TASK=WEEKLY
;;
--monthly|-m)
verbose "running monthly backup"
TASK=MONTHLY
;;
--yearly|-y)
verbose "running yearly backup"
TASK=YEARLY
;;
esac
shift
done
#
#
# create the blank email report
create_tmp_email
#
if test -f $BU_CONF ; then
verbose "Reading default in $BU_CONF"
source $BU_CONF
else
error "Couldn't find or read $BU_CONF"
exit 1
fi
#
# run through the various tasks
verbose "using configuration file $BU_CONF"
# go to the Docker Compose directory where we need to be to run the
# docker-compose commands
verbose "going to $DC_DIR"
OLD_DIR=`pwd`
cd $DC_DIR
#
# a timestamp for logging purposes
STAMP=`date '+%Y-%m-%d_%H-%M-%S'`
#
# generate the filename
INC="BU_FROOT_$TASK"
FILEPART=$BU_FROOT-${!INC}
FILENAME=$FILEPART-$STAMP.sql
FILEPATH=$BU_DIR/$FILENAME
#
# delete stale backups
TO_KEEP="BU_TO_KEEP_$TASK"
delete_old $FILEPART ${!TO_KEEP}
#
message "backing up the selected database into $FILEPATH"
# dump the data into the file
#
do_backup $FILEPATH $TASK
#
message "completed backup"
# compress the backup
message "compressing $FILEPATH"
$GZIP $FILEPATH
# return to where you started from...
cd $OLD_DIR
#
# sent resulting email report
#
send_email_report
exit 0

200
pgdbbackup Normal file
View file

@ -0,0 +1,200 @@
#!/bin/bash
#
# This script dumps the osl db on new.egressive.com
# and pipes the results into an appropriately named/dated
# file...
#
# The following options must be specified in .conf files
# in a directory designated with the -c switch.
#
# # backup directory
# BU_DIR=/storage/warhol/pgsql/daily
# # the database details
# SERVER=localhost
# USER=root
# PASSWORD=
# # names of the databases to backup
# DBS="osl osltest egressive seradigm egressive_accounts"
# # email address to send reports to, and subject
# EMAIL=dlane@egressive.com
# EMAIL_SUBJ="Postgres daily OSL Backup Report"
# # backup identifier
# BU_IDENT=daily
# # number of backups for each db to keep
# BU_TO_KEEP=7
# # output for debugging...
VERBOSE=0
#
# Stuff that should be universal for this install...
# where we can find this app...
MAIN_DIR=/etc/pgdbbackup
# determine today's date
DATE=`date '+%Y-%m-%d-%a'`
# determine today's date
TIME=`date '+%H-%M-%S'`
# temporary holding point for email
TMP_EMAIL=/tmp/tmp_pgdbbackup_email.$DATE_$TIME
# log file
LOG=/var/log/epgdbbackup.log
#
# Commands
# gzip command
GZIP=`which gzip`
# grep command
GREP=`which grep`
# email program
MAIL=`which mail`
# database dump utility
PGDUMP=`which pg_dump`
DEF_ARGS="-C -d -O -x"
#
# pattern for "ls" command to build list of
# pruneable backup files...
# -1t = 1 column, ordered by time of last mod
PRUNEABLES_CMD="ls -1t"
# function to direct a message...
message() {
#
# a timestamp for logging purposes
TIMESTAMP=`date '+%Y-%m-%d %H:%M.%S'`
echo "$0: $TIMESTAMP $@" >> $LOG
if test -f $TMP_EMAIL ; then
echo "$0: $TIMESTAMP $@" >> $TMP_EMAIL
fi
verbose "$TIMESTAMP $@"
}
# create the temporary email file
create_tmp_email() {
touch $TMP_EMAIL
if test -f $TMP_EMAIL ; then
message "created temporary email $TMP_EMAIL"
else
message "failed to create temporary email $TMP_EMAIL"
fi
}
# send the contents of the temporary file to the
# designated report recipient
send_email_report() {
if test -f $TMP_EMAIL ; then
message "sending email report to $EMAIL"
$MAIL -s "$EMAIL_SUBJ" $EMAIL < $TMP_EMAIL
rm $TMP_EMAIL
if test -f $TMP_EMAIL ; then
message "failed to remove temporary email $TMP_EMAIL"
else
message "successfully removed temporary email $TMP_EMAIL"
fi
message "email report successfully sent"
fi
}
# insert a blank line into the log and on the console
insert_blank() {
echo "" >> $LOG
verbose ""
}
# function to direct a message...
verbose() {
if test $VERBOSE = 1 ; then
echo "$@"
fi
}
#
# delete old backups
delete_old() {
#
if test -n $BU_FROOT && test -n $BU_TO_KEEP ; then
# pattern to search for to build the list...
PATTERN="$BU_DIR/$BU_FROOT-*.*"
# build the list, with the suffix...
PRUNEABLES=`$PRUNEABLES_CMD $PATTERN`
if test "$?" -eq "0" ; then
message "pruning older files based on $PATTERN.$BU_SUFFIX"
message "keeping last $BU_TO_KEEP backups"
#
# set counter
NUM=0
# go through the list of files and remove those we don't want
for PRUNEABLE in $PRUNEABLES
do
NUM=$(($NUM + 1))
if test $NUM -gt $BU_TO_KEEP ; then
message "deleting $PRUNEABLE"
rm $PRUNEABLE 2>&1 > /dev/null
else
message "keeping $PRUNEABLE"
fi
done
fi
else
message "keeping older backups, missing backup_root_filename..."
fi
}
#
# cycle through the command line options
while test $# -ne 0 ; do
case $1 in
--config|-c)
shift # shift from the flag to the value
verbose "setting configuration directory to $1"
BU_CONFDIR=$1
;;
esac
shift
done
#
#
# create the blank email report
create_tmp_email
#
# run through the various configuration files
message "looking for configuration files in $BU_CONFDIR"
if test -d $MAIN_DIR/$BU_CONFDIR ; then
CONFS=`ls -1 $MAIN_DIR/$BU_CONFDIR/*.conf`
message "found these config files: $CONFS"
for CONF in $CONFS
do
if test -f $CONF ; then
message "processing $CONF configuration file"
# source the configuration file
source $CONF
# go to the backup directory
cd $BU_DIR
# set up database access info
export PGUSER=$USER
export PGPASSWORD=$PASSWORD
message "starting backup of databases $DBS"
#
# dump the database
for DB in $DBS
do
# a timestamp for logging purposes
STAMP=`date '+%Y-%m-%d_%H-%M-%S'`
# create a root filename
BU_FROOT=$DB-$BU_IDENT
# delete stale backups
delete_old
# generate the filename
FILENAME=$BU_FROOT-$STAMP.sql
message "backing up database '$DB' into file $FILENAME"
# dump the data into the file
$PGDUMP $DEF_ARGS -h $SERVER -f $FILENAME $DB
message "completed backup of '$DB'"
# compress the backup
message "compressing $FILENAME"
$GZIP $FILENAME
# vaccuum the database
done
message "completed backup of databases: $DBS"
# return to where you started from...
cd $OLDPWD
fi
done
message "read config files in $BU_CONF directory"
else
message "config directory $BU_CONF doesn't exist!"
exit 1
fi
# sent resulting email report
send_email_report
exit 0