From 715b32f38fb773dfd74b540d8c0e3cd52775704c Mon Sep 17 00:00:00 2001
From: Dave Lane <dave@davelane.nz>
Date: Thu, 9 Jun 2022 15:26:20 +1200
Subject: [PATCH] initial commit of containerised MariaDB backup functionality

---
 default-mariadb-docker-compose.conf |  22 +++
 mariadbbackup-docker-compose        | 240 ++++++++++++++++++++++++++++
 pgdbbackup                          | 200 +++++++++++++++++++++++
 3 files changed, 462 insertions(+)
 create mode 100644 default-mariadb-docker-compose.conf
 create mode 100644 mariadbbackup-docker-compose
 create mode 100644 pgdbbackup

diff --git a/default-mariadb-docker-compose.conf b/default-mariadb-docker-compose.conf
new file mode 100644
index 0000000..f8a5161
--- /dev/null
+++ b/default-mariadb-docker-compose.conf
@@ -0,0 +1,22 @@
+#
+# SQL dump backup directory
+#
+BU_DIR=/home/data/mastodon/backup
+#
+# Docker Compose details
+#
+# dir containing the docker-compose.yml
+DC_DIR=/home/docker/[dirname]
+DC_FILE=$DC_DIR/docker-compose.yml
+DC_CONTAINER=mariadb
+DC_DB_NAME_VAR=MOODLE_DB_NAME
+DC_DB_USER_VAR=MOODLE_DB_USER
+DC_DB_PASSWORD_VAR=MOODLE_DB_PASSWORD
+# Command to dump the relevant database(s)
+DUMP_CMD='mysqldump '
+#
+# Reporting
+#
+# email address to send reports to, and subject
+EMAIL=webmaster@oerfoundation.org
+EMAIL_SUBJ="Mastodon (on Open) MariaDB Backup Report"
diff --git a/mariadbbackup-docker-compose b/mariadbbackup-docker-compose
new file mode 100644
index 0000000..6e2d657
--- /dev/null
+++ b/mariadbbackup-docker-compose
@@ -0,0 +1,240 @@
+#!/bin/bash
+#
+# This script dumps the PostgreSQL db in a Docker container
+# managed by Docker Compose using the selected command within
+# the database container and pipes the results into an
+# appropriately named/dated file...
+#
+# It also manages rolling out older backups to avoid filling
+# your storage...
+#
+# Default retention
+BU_TO_KEEP_HOURLY=24
+BU_TO_KEEP_DAILY=7
+BU_TO_KEEP_WEEKLY=4
+BU_TO_KEEP_MONTHLY=12
+BU_TO_KEEP_YEARLY=7
+#
+BU_FROOT=alldbs
+BU_FROOT_HOURLY=hourly
+BU_FROOT_DAILY=daily
+BU_FROOT_WEEKLY=weekly
+BU_FROOT_MONTHLY=monthly
+BU_FROOT_YEARLY=yearly
+#
+# this can be overridden at invocation
+BU_CONF=default-docker-compose.conf
+# # output for debugging...
+VERBOSE=0
+#
+# Stuff that should be universal for this install...
+# where we can find this app...
+MAIN_DIR=/home/data/scripts/mariadbbackup-docker-compose
+# determine today's date
+DATE=`date   '+%Y-%m-%d-%a'`
+# determine today's date
+TIME=`date   '+%H-%M-%S'`
+# temporary holding point for email
+TMP_EMAIL=/tmp/tmp_mariadbbackup_email.$DATE_$TIME
+# log file
+LOG=/var/log/mariadbbackup.log
+#
+# Commands
+# gzip command
+GZIP=`which gzip`
+# grep command
+GREP=`which grep`
+# email program
+MAIL=`which mail`
+# database dump utility
+DC=`which docker-compose`
+#DEF_ARGS="-C -d -O -x"
+#
+# pattern for "ls" command to build list of
+# pruneable backup files...
+# -1t = 1 column, ordered by time of last mod
+PRUNEABLES_CMD="ls -1t"
+# function to direct a message...
+message() {
+  #
+  # a timestamp for logging purposes
+  TIMESTAMP=`date  '+%Y-%m-%d %H:%M.%S'`
+  echo "$0: $TIMESTAMP $@" >> $LOG
+  if test -f $TMP_EMAIL ; then
+    echo "$0: $TIMESTAMP $@" >> $TMP_EMAIL
+  fi
+  verbose "$TIMESTAMP $@"
+}
+# create the temporary email file
+create_tmp_email() {
+  touch $TMP_EMAIL
+  if test -f $TMP_EMAIL ; then
+    message "created temporary email  $TMP_EMAIL"
+  else
+    message "failed to create temporary email  $TMP_EMAIL"
+  fi
+}
+# send the contents of the temporary file to the
+# designated report recipient
+send_email_report() {
+  if test -f $TMP_EMAIL ; then
+    message "sending email report to $EMAIL"
+    $MAIL -s "$EMAIL_SUBJ" $EMAIL < $TMP_EMAIL
+    rm $TMP_EMAIL
+    if test -f $TMP_EMAIL ; then
+      message "failed to remove temporary email  $TMP_EMAIL"
+    else
+      message "successfully removed temporary email  $TMP_EMAIL"
+    fi
+    message "email report successfully sent"
+  fi
+}
+# insert a blank line into the log and on the console
+insert_blank() {
+  echo "" >> $LOG
+  verbose ""
+}
+# function to direct a message...
+verbose() {
+  if test $VERBOSE = 1 ; then
+    echo "$@"
+  fi
+}
+#
+# delete old backups
+delete_old() {
+  #
+  verbose "deleting old files based on $1"
+  # pattern to search for to build the list...
+  PATTERN="$BU_DIR/$1-*.*"
+  # build the list, with the suffix...
+  PRUNEABLES=`$PRUNEABLES_CMD $PATTERN`
+  if  test "$?" -eq "0" ; then
+    message "pruning older files based on $PATTERN"
+    BU_TO_KEEP=$2
+    message "keeping last $BU_TO_KEEP backups"
+    #
+    # set counter
+    NUM=0
+    # go through the list of files and remove those we don't want
+    for PRUNEABLE in $PRUNEABLES
+    do
+      NUM=$(($NUM + 1))
+      if test $NUM -gt $BU_TO_KEEP ; then
+        message "deleting $PRUNEABLE"
+        rm $PRUNEABLE 2>&1 > /dev/null
+      else
+        message "keeping $PRUNEABLE"
+      fi
+    done
+  else
+    message "No files with $PATTERN to delete"
+  fi
+}
+#
+#
+do_backup() {
+   FILE=$1
+   DATABASE="$(get_value DC_DB_NAME_VAR $DC_FILE)"
+   USER="$(get_value DC_DB_USER_VAR $DC_FILE)"
+   PASSWORD="$(get_value DC_DB_PASSWORD_VAR $DC_FILE)"
+   verbose "debug - DB details: USER = $USER, PASSWORD = $PASSWORD, DB = $DATABASE"
+   VER=`$DC exec $DC_CONTAINER mysql --version`
+   verbose "MariaDB version $VER - echoing to backup: $FILE"
+   echo "--" > $FILE
+   echo "-- MariaDB Version: $VER" >> $FILE
+   echo "--" >> $FILE
+   echo "" >> $FILE
+   CMD="$DC exec $DC_CONTAINER mysqldump $DATABASE -u $USER -p$PASSWORD"
+   verbose "doing database dump: $CMD"
+   $CMD >> $FILE
+}
+# $1 variable to look for
+# $2 file to look in
+
+get_value() {
+  echo `grep -m 1 "${1}" $2 | cut -d : -f 2 | cut -d '#' -f 1 | tr -d '[:space:]'`
+}
+
+#
+# cycle through the command line options
+while test $# -ne 0 ; do
+  case $1 in
+    --config|-c)
+      shift # shift from the flag to the value
+      verbose "setting configuration directory to $1"
+      BU_CONF=$1
+    ;;
+    --hourly|-h)
+      verbose "running hourly backup"
+      TASK=HOURLY
+    ;;
+    --daily|-d)
+      verbose "running daily backup"
+      TASK=DAILY
+    ;;
+    --weekly|-w)
+      verbose "running weekly backup"
+      TASK=WEEKLY
+    ;;
+    --monthly|-m)
+      verbose "running monthly backup"
+      TASK=MONTHLY
+    ;;
+    --yearly|-y)
+      verbose "running yearly backup"
+      TASK=YEARLY
+    ;;
+  esac
+  shift
+done
+#
+#
+# create the blank email report
+create_tmp_email
+#
+if test -f $BU_CONF ; then
+    verbose "Reading default in $BU_CONF"
+    source $BU_CONF
+else
+    error "Couldn't find or read $BU_CONF"
+    exit 1
+fi
+#
+# run through the various tasks
+verbose "using configuration file $BU_CONF"
+# go to the Docker Compose directory where we need to be to run the
+# docker-compose commands
+verbose "going to $DC_DIR"
+OLD_DIR=`pwd`
+cd $DC_DIR
+#
+# a timestamp for logging purposes
+STAMP=`date  '+%Y-%m-%d_%H-%M-%S'`
+#
+# generate the filename
+INC="BU_FROOT_$TASK"
+FILEPART=$BU_FROOT-${!INC}
+FILENAME=$FILEPART-$STAMP.sql
+FILEPATH=$BU_DIR/$FILENAME
+#
+# delete stale backups
+TO_KEEP="BU_TO_KEEP_$TASK"
+delete_old $FILEPART ${!TO_KEEP}
+#
+message "backing up the selected database into $FILEPATH"
+# dump the data into the file
+#
+do_backup $FILEPATH $TASK
+#
+message "completed backup"
+# compress the backup
+message "compressing $FILEPATH"
+$GZIP $FILEPATH
+# return to where you started from...
+cd $OLD_DIR
+#
+# sent resulting email report
+#
+send_email_report
+exit 0
diff --git a/pgdbbackup b/pgdbbackup
new file mode 100644
index 0000000..ccae8a7
--- /dev/null
+++ b/pgdbbackup
@@ -0,0 +1,200 @@
+#!/bin/bash
+#
+# This script dumps the osl db on new.egressive.com
+# and pipes the results into an appropriately named/dated
+# file...
+#
+# The following options must be specified in .conf files
+# in a directory designated with the -c switch.
+#
+# # backup directory
+# BU_DIR=/storage/warhol/pgsql/daily
+# # the database details
+# SERVER=localhost
+# USER=root
+# PASSWORD=
+# # names of the databases to backup
+# DBS="osl osltest egressive seradigm egressive_accounts"
+# # email address to send reports to, and subject
+# EMAIL=dlane@egressive.com
+# EMAIL_SUBJ="Postgres daily OSL Backup Report"
+# # backup identifier
+# BU_IDENT=daily
+# # number of backups for each db to keep
+# BU_TO_KEEP=7
+# # output for debugging...
+VERBOSE=0
+#
+# Stuff that should be universal for this install...
+# where we can find this app...
+MAIN_DIR=/etc/pgdbbackup
+# determine today's date
+DATE=`date   '+%Y-%m-%d-%a'`
+# determine today's date
+TIME=`date   '+%H-%M-%S'`
+# temporary holding point for email
+TMP_EMAIL=/tmp/tmp_pgdbbackup_email.$DATE_$TIME
+# log file
+LOG=/var/log/epgdbbackup.log
+#
+# Commands
+# gzip command
+GZIP=`which gzip`
+# grep command
+GREP=`which grep`
+# email program
+MAIL=`which mail`
+# database dump utility
+PGDUMP=`which pg_dump`
+DEF_ARGS="-C -d -O -x"
+#
+# pattern for "ls" command to build list of
+# pruneable backup files...
+# -1t = 1 column, ordered by time of last mod
+PRUNEABLES_CMD="ls -1t"
+# function to direct a message...
+message() {
+  #
+  # a timestamp for logging purposes
+  TIMESTAMP=`date  '+%Y-%m-%d %H:%M.%S'`
+  echo "$0: $TIMESTAMP $@" >> $LOG
+  if test -f $TMP_EMAIL ; then
+    echo "$0: $TIMESTAMP $@" >> $TMP_EMAIL
+  fi
+  verbose "$TIMESTAMP $@"
+}
+# create the temporary email file
+create_tmp_email() {
+  touch $TMP_EMAIL
+  if test -f $TMP_EMAIL ; then
+    message "created temporary email  $TMP_EMAIL"
+  else
+    message "failed to create temporary email  $TMP_EMAIL"
+  fi
+}
+# send the contents of the temporary file to the
+# designated report recipient
+send_email_report() {
+  if test -f $TMP_EMAIL ; then
+    message "sending email report to $EMAIL"
+    $MAIL -s "$EMAIL_SUBJ" $EMAIL < $TMP_EMAIL
+    rm $TMP_EMAIL
+    if test -f $TMP_EMAIL ; then
+      message "failed to remove temporary email  $TMP_EMAIL"
+    else
+      message "successfully removed temporary email  $TMP_EMAIL"
+    fi
+    message "email report successfully sent"
+  fi
+}
+# insert a blank line into the log and on the console
+insert_blank() {
+  echo "" >> $LOG
+  verbose ""
+}
+# function to direct a message...
+verbose() {
+  if test $VERBOSE = 1 ; then
+    echo "$@"
+  fi
+}
+#
+# delete old backups
+delete_old() {
+  #
+  if test -n $BU_FROOT && test -n $BU_TO_KEEP ; then
+    # pattern to search for to build the list...
+    PATTERN="$BU_DIR/$BU_FROOT-*.*"
+    # build the list, with the suffix...
+    PRUNEABLES=`$PRUNEABLES_CMD $PATTERN`
+    if  test "$?" -eq "0" ; then
+      message "pruning older files based on $PATTERN.$BU_SUFFIX"
+      message "keeping last $BU_TO_KEEP backups"
+      #
+      # set counter
+      NUM=0
+      # go through the list of files and remove those we don't want
+      for PRUNEABLE in $PRUNEABLES
+      do
+        NUM=$(($NUM + 1))
+        if test $NUM -gt $BU_TO_KEEP ; then
+          message "deleting $PRUNEABLE"
+          rm $PRUNEABLE 2>&1 > /dev/null
+        else
+          message "keeping $PRUNEABLE"
+        fi
+      done
+    fi
+  else
+    message "keeping older backups, missing backup_root_filename..."
+  fi
+}
+
+#
+# cycle through the command line options
+while test $# -ne 0 ; do
+  case $1 in
+    --config|-c)
+      shift # shift from the flag to the value
+      verbose "setting configuration directory to $1"
+      BU_CONFDIR=$1
+    ;;
+  esac
+  shift
+done
+#
+#
+# create the blank email report
+create_tmp_email
+#
+# run through the various configuration files
+message "looking for configuration files in $BU_CONFDIR"
+if test -d $MAIN_DIR/$BU_CONFDIR ; then
+  CONFS=`ls -1 $MAIN_DIR/$BU_CONFDIR/*.conf`
+  message "found these config files: $CONFS"
+  for CONF in $CONFS
+  do
+    if test -f $CONF ; then
+      message "processing $CONF configuration file"
+      # source the configuration file
+      source $CONF
+      # go to the backup directory
+      cd $BU_DIR
+      # set up database access info
+      export PGUSER=$USER
+      export PGPASSWORD=$PASSWORD
+      message "starting backup of databases $DBS"
+      #
+      # dump the database
+      for DB in $DBS
+      do
+        # a timestamp for logging purposes
+        STAMP=`date  '+%Y-%m-%d_%H-%M-%S'`
+        # create a root filename
+        BU_FROOT=$DB-$BU_IDENT
+        # delete stale backups
+        delete_old
+        # generate the filename
+        FILENAME=$BU_FROOT-$STAMP.sql
+        message "backing up database '$DB' into file $FILENAME"
+        # dump the data into the file
+        $PGDUMP $DEF_ARGS -h $SERVER -f $FILENAME $DB
+        message "completed backup of '$DB'"
+        # compress the backup
+        message "compressing $FILENAME"
+        $GZIP $FILENAME
+        # vaccuum the database
+      done
+      message "completed backup of databases: $DBS"
+      # return to where you started from...
+      cd $OLDPWD
+    fi
+  done
+  message "read config files in $BU_CONF directory"
+else
+  message "config directory $BU_CONF doesn't exist!"
+  exit 1
+fi
+# sent resulting email report
+send_email_report
+exit 0