commit 01334fae094574b4311b0bc377376d536f44f8c8 Author: Dave Lane Date: Tue May 2 02:01:02 2023 +0000 initial commit of this new project diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..030886d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +site.conf diff --git a/egdbback b/egdbback new file mode 100755 index 0000000..1d5f68d --- /dev/null +++ b/egdbback @@ -0,0 +1,473 @@ +#!/bin/bash +# +# egdbback, copyright 2005-2008 Egressive Limited, http://egressive.com +# +# this script runs periodic database backups... +# +#========================================== +# The Variables - these must be define +# before they're referenced +#========================================== +VERSION="0.3" +EGDB_NAME=`basename $0` +EGDB_DIR=/home/data/scripts/egdbback +EGDB_CMD=$EGDB_DIR/egdbback +SITE_CONF="$EGDB_DIR/site.conf" +# the appropriate mysql paths +MYDIR=/var/lib/mysql +# +# this provides values for MACHINE_NAME and EMAIL +. $SITE_CONF +# +LOGS=/var/log +LOG=$LOGS/egdbback.log +#========================================== +# Defaults - these shouldn't need changing +#========================================== +PERIODS="hourly daily weekly monthly yearly" +hourlyNUM="24" +dailyNUM="7" +weeklyNUM="4" +monthlyNUM="12" +yearlyNUM="7" + +hourlyNEXT="daily" +dailyNEXT="weekly" +weeklyNEXT="monthly" +monthlyNEXT="yearly" + +# required programs +MAIL=`which mail` +GREP=`which grep` +LS=`which ls` +ID=`which id` +DATE=`which date` +DF=`which df` +GZIP=`which gzip` +GZIPSUF=gz +RM=`which rm` +LN=`which ln` +CUT=`which cut` +AWK=`which awk` +CP=`which cp` +# determine today's date +TODAY=`$DATE '+%Y-%m-%d-%a'` +# determine today's date +NOW=`$DATE '+%H-%M-%S'` +# a timestamp for logging purposes +TIMESTAMP=`$DATE '+%Y-%m-%d %H:%M.%S'` +# temporary holding point for email +TMP_DIR=/tmp +TMP_EMAIL=$TMP_DIR/$EGDB_NAME"_tmp_email_"$TODAY.$TIME +# +#========================================== +# MySQL vars... +MYTMP=$MYBKP/tmp-$DATE +MYDUMP=`which mysqldump` +MYSQL=`which mysql` +MYSHOW=`which mysqlshow` +#========================================== +# The Functions - these have to be defined +# before they're called! Note, refer +# to them without the "()" and provide +# up to one argument, referred to as "$@" +# in the function... +#========================================== +# +# function to direct a message... +message() { + # a timestamp for logging purposes + timestamp + if test -w $LOG ; then + echo "$EGDB_NAME: $TIMESTAMP $@" >> $LOG + fi + if test -w $TMP_EMAIL ; then + echo "$EGDB_NAME: $TIMESTAMP $@" >> $TMP_EMAIL + fi + verbose "$TIMESTAMP $@" +} +# +# function to direct a message... +verbose() { + if test $VERBOSE ; then + echo "$@" + fi +} +# +# insert a blank line into the log and on the console +insert_blank() { + echo "" >> $TMP_EMAIL + verbose "" +} +# +# update date and time info.. +today() { + # determine today's date + TODAY=`$DATE '+%Y-%m-%d-%a'` +} +# +# +now() { + # determine today's date + NOW=`$DATE '+%H-%M-%S'` +} +# +# +timestamp() { + # a timestamp for logging purposes + TIMESTAMP=`$DATE '+%Y-%m-%d %H:%M.%S'` +} +# +# create the temporary email file +create_tmp_email() { + if test -d $TMP_DIR ; then + if test -w $TMP_DIR ; then + touch $TMP_EMAIL 2>&1 + else + error "Email tmp directory $TMP_DIR is not writable" + fi + else + error "Email tmp directory $TMP_DIR does not exist" + fi + + if test -w $TMP_EMAIL ; then + message "created temporary email $TMP_EMAIL" + else + error "Failed to create temporary email $TMP_EMAIL" + fi +} +# +# send the contents of the temporary file to the +# designated report recipient +send_email_report() { + if test -f $TMP_EMAIL ; then + message "sending email report to $EMAIL_TO" + if test $ERROR_STATUS == 1 ; then + EMAIL_SUBJ="[ERROR] $MACHINE_NAME $DEFAULT_EMAIL_SUBJ" + else + EMAIL_SUBJ="[SUCCESS] $MACHINE_NAME $DEFAULT_EMAIL_SUBJ" + fi + # check space again to see how close things are to full + + message "Printing disk space for reference purposes:" + DISK_SPACE=`$DF` + message "$DISK_SPACE" + # send it to each email address in the list... + for EMAIL_ADD in $EMAIL_TO + do + RES=`$MAIL -s "$EMAIL_SUBJ" $EMAIL_ADD < $TMP_EMAIL` + if test -z $RES ; then + if test $ERROR_STATUS == 1 ; then + message "Error email report successfully sent to $EMAIL_ADD" + else + message "Email report successfully sent to $EMAIL_ADD" + fi + else + if ! test $ERROR_STATUS == 1 ; then + error "Email report send to $EMAIL_ADD failed with this message: $RES" + fi + fi + done + if test -w $TMP_EMAIL ; then + $RM $TMP_EMAIL 2>&1 + else + if ! test $ERROR_STATUS == 1 ; then + error "Failed to remove email message file, $TMP_EMAIL: permission denied." + fi + fi + if test -f $TMP_EMAIL ; then + error "Failed to remove email message file $TMP_EMAIL for an unknown reason" + else + message "successfully removed temporary email $TMP_EMAIL" + fi + else + if ! test $ERROR_STATUS == 1 ; then + error "Email message file, $TMP_EMAIL, does not exist." + fi + fi +} +# +# function to direct an error... +error() { + # + # recognise that the system experienced + # an error and send out a special email, and halt + # the program! + timestamp + ERROR_STATUS=1 + # + # get current user details + USER_DETAILS=`$ID` + if test -w $LOG ; then + echo "$EGDB_NAME: **ERROR** $TIMESTAMP $@" >> $LOG + echo "$EGDB_NAME: user details - $USER_DETAILS" >> $LOG + fi + if test -w $TMP_EMAIL ; then + echo "$EGDB_NAME: **ERROR** $TIMESTAMP $@" >> $TMP_EMAIL + echo " user details: $USER_DETAILS" >> $LOG + fi + verbose "$TIMESTAMP **ERROR** $@" + verbose " user details: $USER_DETAILS" + send_email_report + exit 1 +} +# +# delete old backups +delete_old() { + for PERIOD in $PERIODS + do + BU_TO_KEEP=$((${PERIOD}NUM)) + PATTERN="$PERIOD-$DB-*.*" + message "keeping $BU_TO_KEEP files for $PERIOD based on the pattern $PATTERN" + PRUNEABLES=`$LS -t1 $BU_DIR/$PATTERN` + #echo "pruneable files: $PRUNEABLES" + NUM=0 + for FILE in $PRUNEABLES + do + NUM=$(($NUM + 1)) + if test $NUM -gt $BU_TO_KEEP ; then + message "removing $FILE ($NUM/$BU_TO_KEEP)" + $RM $FILE + fi + done + done +} +# +# convert last monthly into daily, daily->weekly, weekly->monthly, monthly->yearly +roll_over() { + # First figure out if this backup is a "roll" one + # by conducting a few tests + hourly_ROLL=0 + daily_ROLL=0 + weekly_ROLL=0 + monthly_ROLL=0 + message "checking for roll overs" + # hour of the day " 0"-"23" + TEST=`$DATE '+%k'` + #TEST=`$DATE '+%k' --date="2008-01-01 00:00"` + if test $TEST -eq " 0" ; then # note the "[space]0" + # if test "0" -eq "0" ; then + message "time ($TEST) to create a new daily backup!" + hourly_ROLL=1 + fi + message "new day? $TEST" + # day of the week 0-6 + TEST=`$DATE '+%w'` + if test $TEST -eq "0" && test $hourly_ROLL == 1 ; then + message "time ($TEST) to create a new weekly backup!" + daily_ROLL=1 + fi + message "new week? $TEST" + # day of month 0-31ish + TEST=`$DATE '+%d'` + # should also be at the start of a day... not necessarily a week. + if test $TEST -eq "01" && test $hourly_ROLL == 1 ; then + message "time ($TEST) to create a new monthly backup!" + weekly_ROLL=1 + fi + message "new month? $TEST" + # day of year 0-366ish + TEST=`$DATE '+%j'` + # should also be at the start of a day... not necessarily a week. + if test $TEST -eq "001" && test $hourly_ROLL == 1 ; then + message "time ($TEST) to create a new yearly backup!" + monthly_ROLL=1 + fi + message "new year? $TEST" + # organise the actual rolling of files. + ROLL=0 + for PERIOD in $PERIODS + do + # do we roll this period's backup? + ROLL=$((${PERIOD}_ROLL)) + echo "checking $PERIOD for rollover: $ROLL" + if test $ROLL -eq "1" ; then + echo "rollover for $PERIOD" + # Use the dynamic variable approach... + VAR=${PERIOD}NEXT + # ...to set value of NEXT to the *value* of $VAR + eval NEXT=\$$VAR + if ! test -z $NEXT; then + echo "rolling $PERIOD to $NEXT" + # get the root name for the most recent $PERIOD file + PATTERN="$FROOT-*.*" + PERIOD_FILES=`$LS -qt1 $BU_DIR/$PATTERN` + RET=$? + echo "\$?=$RET" + # only proceed with the copy if there's a suitable file + # ls return 0 if it finds files, 1 or 2 if not + if test $RET == 0 ; then + for FILE in $PERIOD_FILES + do + echo "copying $FILE" + break + done + # the echo "" is required to launch awk properly... + NEW_FILE=`echo "" | $AWK -v fn=$FILE -v nxt=$NEXT '{ + split(fn,filepath,"/"); + num=0; for (val in filepath) num++; + filename = filepath[num]; + split(filename,filenameparts,"-"); + filecut = length(filenameparts[1]); + pathcut = length(filename); + path = substr(fn,1,length(fn)-pathcut); + subname = substr(filename, filecut + 1); + print path nxt subname; }'` + echo "to $NEW_FILE" + $CP $FILE $NEW_FILE + else + echo "no suitable file to copy!" + fi + else + echo "there is no 'next' for $PERIOD" + fi + else + echo "no rollover for $PERIOD this time" + fi + done +} +# +# Make sure there's sufficient disk space (or at least that it's not full!) +check_space() { + RES=`$DF -h` + #message "$RES" + TEST=0 + PCENT="100%" + TEST=`echo "$RES" | $GREP -c "$PCENT"` + if test $TEST == 1 ; then + PART_LINE=`echo "$RES" | $GREP "$PCENT"` + # get the device name which is showing $PCENT + TEST=`echo $PART_LINE | $CUT -f 1 -d ' '` + # if it has a partition number in the name, it's probably a block + # device rather than, say, a CDROM... + TEST2=`echo $TEST | $GREP -c [0-9]` + echo "Test2 = $TEST2" + if test $TEST2 == 1 ; then + error "Partition $TEST is full - detail: $PART_LINE" + else + message "false alarm - it's only the $TEST volume..." + fi + fi +} +# +# this is where we do all the backup-related stuff - what ebu used to do... +do_backup() { + # first, find out what dbs there are + DBLIST=`$MYSHOW -u $USER -p$PASSWORD | $AWK '{ if ($2!~/^$/&&$2!="Databases") { print $2;}}'` + #message "List of databases: $DBLIST" + # for each db in the list, create a compressed backup in the right place + message "creating a backup of the MySQL databases on $MACHINE_NAME..." + for DB in $DBLIST + do + FROOT=hourly-$DB + # work out which backups should be turned into dailies, weeklies, monthlies, etc. + roll_over + # we only create hourlies + now + today + FNAME=$FROOT-$TODAY.$NOW + # --opt is same as --add-drop-table --add-locks --all --extended-insert --quick --lock-tables + CMD="$MYDUMP -u $USER -p$PASSWORD --opt $DB" + # create the new backup! + if test $DRY_RUN == 1 ; then + message "I would be doing this (if it was not a dry run):" + message "$CMD > $BU_DIR/$FNAME" + else + message "backing up database $DB..." + message "running $MYDUMP with user $USER, creating $BU_DIR/$FNAME" + $CMD > $BU_DIR/$FNAME + if test -f $BU_DIR/$FNAME ; then + $GZIP $BU_DIR/$FNAME + else + error "backup sql file, $BU_DIR/$FNAME, doesn't exist!" + fi + fi + # delete old backups, so that only BU_TO_KEEP of them are stored at once + delete_old + done +} +#======================================== +# The Error Checking +#======================================== +# +# first, check that all the necessary files and stuff are there and usable +# +# if the log doesn't exist, create it +if ! test -f $LOG ; then + message "creating non-existent log file $LOG" + RET=`touch $LOG 2>&1 /dev/null` + TEST=`echo $RET | $GREP -c "Permission denied" -` + if test $TEST == 1 ; then + error "Failed to create log file $LOG, user cannot write to that directory" + fi +fi +# checking whether it's there an is writable +if ! test -w $LOG ; then + error "Log file $LOG not writable" +fi + +#======================================== +# The Functional Part of the Script +#======================================== +# set variable defaults +ERROR_STATUS=0 # initially assume no errors +VERBOSE=0 # initially assume quiet output +DRY_RUN=0 # assume this is not a dry run... +MODE=help +# process any arguments +while test $# -ne 0 ; do # while there are arguments + message "argument: $1" + case $1 in + --verbose|-v) + verbose "setting verbostity to ON" + VERBOSE=1 + ;; + --dryrun) + verbose "setting dryrun to ON" + DRY_RUN=1 + ;; + --run|-r) + verbose "setting run to ON" + MODE=run + ;; + --help|-h|-?) + verbose "setting mode to HELP" + MODE=help + ;; + *) + verbose "unknown option: $1" + message "unknown option: $1" + MODE=help + ;; + esac + shift +done +if ! test $MODE == "help" ; then + MODE=run +fi +# +# set things in motion based on the script arguments +case $MODE in + run) + create_tmp_email + do_backup + # put a space in the email to separate tasks + insert_blank + # + # send the resulting email + send_email_report + ;; + help) + echo "" + echo "$EGDB_NAME, version $VERSION, copyright 2005-7 Egressive Ltd, http://egressive.com" + echo "===================" + echo "usage: $EGDB_NAME" + echo "== options ==" + echo "-r or --run - actually run the backups!" + echo "-v or --verbose - give extra feedback on progress to stdout" + echo "-c or --configroot config_filename - root for configuration filenames" + echo "--dryrun - do everything else, but *don't* delete anything or run the backup" + echo "" + ;; +esac + diff --git a/egdbback-cron b/egdbback-cron new file mode 100755 index 0000000..d3aff1a --- /dev/null +++ b/egdbback-cron @@ -0,0 +1,4 @@ +# run the egdb backup programs every day to backup our MySQL database. +# hourlies - every hour at 5 past the hour... +5 * * * * root /etc/egscripts/egdbback/egdbback --run + diff --git a/logrotate.d/egdbback b/logrotate.d/egdbback new file mode 100644 index 0000000..d6fceaa --- /dev/null +++ b/logrotate.d/egdbback @@ -0,0 +1,7 @@ +/var/log/egdbbackup*.log { + monthly + missingok + rotate 7 + compress + notifempty +} diff --git a/site.conf-sample b/site.conf-sample new file mode 100644 index 0000000..803194e --- /dev/null +++ b/site.conf-sample @@ -0,0 +1,17 @@ +# +# Defaults +# +# replace the [tokens] below! +# +# default email address to which to send backup reports +EMAIL=[webmaster@example.com] +# +# default email subject +MACHINE_NAME=`hostname` +# +# directory for the backups +BU_DIR=[path/to/backups] +# These are specific to the MySQL install on this machine +HOST= # blank if localhost +USER=[backup user] +PASSWORD=[backup user password]