From c35535139450d71cb6f2ffa24860176b9b89c74c Mon Sep 17 00:00:00 2001 From: Dave Lane Date: Thu, 9 Jun 2022 15:57:32 +1200 Subject: [PATCH 01/11] updating .conf to .conf-sample and tweaking the mariadb scirpt --- ...ker-compose.conf => default-mariadb-docker-compose.conf-sample | 0 mariadbbackup-docker-compose | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename default-mariadb-docker-compose.conf => default-mariadb-docker-compose.conf-sample (100%) mode change 100644 => 100755 mariadbbackup-docker-compose diff --git a/default-mariadb-docker-compose.conf b/default-mariadb-docker-compose.conf-sample similarity index 100% rename from default-mariadb-docker-compose.conf rename to default-mariadb-docker-compose.conf-sample diff --git a/mariadbbackup-docker-compose b/mariadbbackup-docker-compose old mode 100644 new mode 100755 From ef6958a3b9cf6b6cb8cc8b445d0e60c498056897 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 9 Jun 2022 05:53:37 +0000 Subject: [PATCH 02/11] fixing the parameter extraction and mysqldump process --- mariadbbackup-docker-compose | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mariadbbackup-docker-compose b/mariadbbackup-docker-compose index 6e2d657..4deaa07 100755 --- a/mariadbbackup-docker-compose +++ b/mariadbbackup-docker-compose @@ -23,9 +23,9 @@ BU_FROOT_MONTHLY=monthly BU_FROOT_YEARLY=yearly # # this can be overridden at invocation -BU_CONF=default-docker-compose.conf +BU_CONF=default-mariadb-docker-compose.conf # # output for debugging... -VERBOSE=0 +VERBOSE=1 # # Stuff that should be universal for this install... # where we can find this app... @@ -135,9 +135,9 @@ delete_old() { # do_backup() { FILE=$1 - DATABASE="$(get_value DC_DB_NAME_VAR $DC_FILE)" - USER="$(get_value DC_DB_USER_VAR $DC_FILE)" - PASSWORD="$(get_value DC_DB_PASSWORD_VAR $DC_FILE)" + DATABASE="$(get_value $DC_DB_NAME_VAR $DC_FILE)" + USER="$(get_value $DC_DB_USER_VAR $DC_FILE)" + PASSWORD="$(get_value $DC_DB_PASSWORD_VAR $DC_FILE)" verbose "debug - DB details: USER = $USER, PASSWORD = $PASSWORD, DB = $DATABASE" VER=`$DC exec $DC_CONTAINER mysql --version` verbose "MariaDB version $VER - echoing to backup: $FILE" @@ -191,13 +191,13 @@ done # # # create the blank email report -create_tmp_email +#create_tmp_email # if test -f $BU_CONF ; then verbose "Reading default in $BU_CONF" source $BU_CONF else - error "Couldn't find or read $BU_CONF" + message "Couldn't find or read $BU_CONF" exit 1 fi # @@ -236,5 +236,5 @@ cd $OLD_DIR # # sent resulting email report # -send_email_report +#send_email_report exit 0 From ed83575decde9b12a951d1b36612d5ca6afaa1de Mon Sep 17 00:00:00 2001 From: root Date: Thu, 9 Jun 2022 05:59:00 +0000 Subject: [PATCH 03/11] changed the default backup file root name to be based on the actual database name defined for the container --- mariadbbackup-docker-compose | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mariadbbackup-docker-compose b/mariadbbackup-docker-compose index 4deaa07..15eb7be 100755 --- a/mariadbbackup-docker-compose +++ b/mariadbbackup-docker-compose @@ -15,7 +15,7 @@ BU_TO_KEEP_WEEKLY=4 BU_TO_KEEP_MONTHLY=12 BU_TO_KEEP_YEARLY=7 # -BU_FROOT=alldbs +#BU_FROOT=alldbs BU_FROOT_HOURLY=hourly BU_FROOT_DAILY=daily BU_FROOT_WEEKLY=weekly @@ -211,6 +211,9 @@ cd $DC_DIR # # a timestamp for logging purposes STAMP=`date '+%Y-%m-%d_%H-%M-%S'` +# +# set the root based on the name of the database... +BU_FROOT="$(get_value $DC_DB_NAME_VAR $DC_FILE)" # # generate the filename INC="BU_FROOT_$TASK" From 1bef97e05cdc362ed43e4019f4a3f22984601cfc Mon Sep 17 00:00:00 2001 From: Dave Lane Date: Thu, 9 Jun 2022 07:21:28 +0000 Subject: [PATCH 04/11] Update README.md --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 9255ffb..81e6964 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,13 @@ The purpose of this script is to backup the contents of Docker container-based database (either PostgreSQL or MariaDB) to a local 'dump' file (SQL format), which is then compressed to conserve disk space. The system is designed to be run regularly (perhaps hourly, or daily) via cron, and to keep a pre-determined (via configuration) number of past, deleting excess backup files to conserve disk space. + +## To install + +To install this script, by our convention on Docker Compose hosts, we clone it into `/home/data/scripts` (creating the directory if necessary), via `git clone git@git.oeru.org:oeru/docker-compose-dbbackup.git`. + +Then, in that directory, we copy the relevant '\*.conf-sample' configuration file and copy it to a '\*.conf' file and adjust its values. You can try running the backup to test it by running (for example) + +`./mariadbbackup-docker-compose --hourly` + +which should, if the configuration is valid, create an hourly database backup (starting with the database name with '-hourly' affixed, e.g. ) From 958d0070538e5c8a1d4a33ee22693a1280463a45 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 17 Jun 2022 02:10:29 +0000 Subject: [PATCH 05/11] tweak to specifying config file handling --- mariadbbackup-docker-compose | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mariadbbackup-docker-compose b/mariadbbackup-docker-compose index 15eb7be..340b4e8 100755 --- a/mariadbbackup-docker-compose +++ b/mariadbbackup-docker-compose @@ -25,7 +25,7 @@ BU_FROOT_YEARLY=yearly # this can be overridden at invocation BU_CONF=default-mariadb-docker-compose.conf # # output for debugging... -VERBOSE=1 +VERBOSE=0 # # Stuff that should be universal for this install... # where we can find this app... @@ -162,7 +162,7 @@ while test $# -ne 0 ; do case $1 in --config|-c) shift # shift from the flag to the value - verbose "setting configuration directory to $1" + verbose "setting configuration file to $1" BU_CONF=$1 ;; --hourly|-h) From 258715bcd357f890f7949006ecae6626512923da Mon Sep 17 00:00:00 2001 From: Dave Lane Date: Wed, 24 Aug 2022 14:20:24 +1200 Subject: [PATCH 06/11] fixed backups for the updated docker compose v2 command --- dbbackup-docker-compose | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dbbackup-docker-compose b/dbbackup-docker-compose index a44cc57..f0393a4 100755 --- a/dbbackup-docker-compose +++ b/dbbackup-docker-compose @@ -29,7 +29,7 @@ VERBOSE=0 # # Stuff that should be universal for this install... # where we can find this app... -#MAIN_DIR=`pwd` +MAIN_DIR=/home/data/scripts/pgdbbackup # determine today's date DATE=`date '+%Y-%m-%d-%a'` # determine today's date @@ -47,7 +47,7 @@ GREP=`which grep` # email program MAIL=`which mail` # database dump utility -DC=`which docker-compose` +DC=`which docker` #DEF_ARGS="-C -d -O -x" # # pattern for "ls" command to build list of @@ -136,13 +136,13 @@ delete_old() { do_backup() { FILE=$1 CONTAINER=$2 - VER=`$DC exec -T $CONTAINER pg_config --version` + VER=`$DC compose exec $CONTAINER pg_config --version` verbose "Postgres version $VER - echoing to backup: $FILE" echo "--" > $FILE echo "-- PostgreSQL Version: $VER" >> $FILE echo "--" >> $FILE echo "" >> $FILE - CMD="$DC exec -T $CONTAINER $DUMP_CMD" + CMD="$DC compose exec $CONTAINER $DUMP_CMD" verbose "doing database dump: $CMD" $CMD >> $FILE } From cf1a69e1c7b900cddc483abaa123bd7945410030 Mon Sep 17 00:00:00 2001 From: Dave Lane Date: Wed, 24 Aug 2022 05:06:17 +0000 Subject: [PATCH 07/11] changed the default dir name to match the project name to be clone-friendly, fixed the paths in the cron file and a typo that meant yearlies wouldn't run --- dbbackup-cron | 10 +++++----- dbbackup-docker-compose | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbbackup-cron b/dbbackup-cron index 4b50a2c..12be628 100644 --- a/dbbackup-cron +++ b/dbbackup-cron @@ -4,17 +4,17 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin # run system backups # # hourly - at 5 minutes past mon-sat -25 * * * * root /home/data/mastodon.oeru.org/scripts/dbbackup/dbbackup-docker-compose -c /home/data/mastodon.oeru.org/scripts/dbbackup/mastodon.conf --hourly +25 * * * * root /home/data/scripts/dbbackup-docker-compose/dbbackup-docker-compose -c /home/data/scripts/dbbackup-docker-compose/default-docker-compose.conf --hourly # # daily - at 7:30 pm, mon-sat -30 19 * * * root /home/data/mastodon.oeru.org/scripts/dbbackup/dbbackup-docker-compose -c /home/data/mastodon.oeru.org/scripts/dbbackup/mastodon.conf --daily +30 19 * * * root /home/data/scripts/dbbackup-docker-compose/dbbackup-docker-compose -c /home/data/scripts/dbbackup-docker-compose/default-docker-compose.conf --daily # # weekly - at 7:30 pm, sun -30 19 * * sun root /home/data/mastodon.oeru.org/scripts/dbbackup/dbbackup-docker-compose -c /home/data/mastodon.oeru.org/scripts/dbbackup/mastodon.conf --weekly +30 19 * * sun root /home/data/scripts/dbbackup-docker-compose/dbbackup-docker-compose -c /home/data/scripts/dbbackup-docker-compose/default-docker-compose.conf --weekly # # monthly - at 8:30 pm, on the first of the last of the month -30 20 1 * * root /home/data/mastodon.oeru.org/scripts/dbbackup/dbbackup-docker-compose -c /home/data/mastodon.oeru.org/scripts/dbbackup/mastodon.conf --monthly +30 20 1 * * root /home/data/scripts/dbbackup-docker-compose/dbbackup-docker-compose -c /home/data/scripts/dbbackup-docker-compose/default-docker-compose.conf --monthly # # yearly - at 8:30 pm, on the first of January. -30 20 1 1 * root /home/data/mastodon.oeru.org/scripts/dbbackup/dbbackup-docker-compose -c /home/data/mastodon.oeru.org/scripts/dbbackup/mastodon.conf--yearly +30 20 1 1 * root /home/data/scripts/dbbackup-docker-compose/dbbackup-docker-compose -c /home/data/scripts/dbbackup-docker-compose/default-docker-compose.conf --yearly diff --git a/dbbackup-docker-compose b/dbbackup-docker-compose index f0393a4..0b8bcc3 100755 --- a/dbbackup-docker-compose +++ b/dbbackup-docker-compose @@ -29,7 +29,7 @@ VERBOSE=0 # # Stuff that should be universal for this install... # where we can find this app... -MAIN_DIR=/home/data/scripts/pgdbbackup +MAIN_DIR=/home/data/scripts/dbbackup-docker-compose # determine today's date DATE=`date '+%Y-%m-%d-%a'` # determine today's date From b07e2f1ff2e55e9d695c9f9978ff7695588925bd Mon Sep 17 00:00:00 2001 From: Dave Lane Date: Wed, 24 Aug 2022 08:22:53 +0000 Subject: [PATCH 08/11] trying again, this time with the actual project name --- dbbackup-cron | 10 +++++----- dbbackup-docker-compose | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbbackup-cron b/dbbackup-cron index 12be628..680824b 100644 --- a/dbbackup-cron +++ b/dbbackup-cron @@ -4,17 +4,17 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin # run system backups # # hourly - at 5 minutes past mon-sat -25 * * * * root /home/data/scripts/dbbackup-docker-compose/dbbackup-docker-compose -c /home/data/scripts/dbbackup-docker-compose/default-docker-compose.conf --hourly +25 * * * * root /home/data/scripts/docker-compose-dbbackup/dbbackup-docker-compose -c /home/data/scripts/docker-compose-dbbackup/default-docker-compose.conf --hourly # # daily - at 7:30 pm, mon-sat -30 19 * * * root /home/data/scripts/dbbackup-docker-compose/dbbackup-docker-compose -c /home/data/scripts/dbbackup-docker-compose/default-docker-compose.conf --daily +30 19 * * * root /home/data/scripts/docker-compose-dbbackup/dbbackup-docker-compose -c /home/data/scripts/docker-compose-dbbackup/default-docker-compose.conf --daily # # weekly - at 7:30 pm, sun -30 19 * * sun root /home/data/scripts/dbbackup-docker-compose/dbbackup-docker-compose -c /home/data/scripts/dbbackup-docker-compose/default-docker-compose.conf --weekly +30 19 * * sun root /home/data/scripts/docker-compose-dbbackup/dbbackup-docker-compose -c /home/data/scripts/docker-compose-dbbackup/default-docker-compose.conf --weekly # # monthly - at 8:30 pm, on the first of the last of the month -30 20 1 * * root /home/data/scripts/dbbackup-docker-compose/dbbackup-docker-compose -c /home/data/scripts/dbbackup-docker-compose/default-docker-compose.conf --monthly +30 20 1 * * root /home/data/scripts/docker-compose-dbbackup/dbbackup-docker-compose -c /home/data/scripts/docker-compose-dbbackup/default-docker-compose.conf --monthly # # yearly - at 8:30 pm, on the first of January. -30 20 1 1 * root /home/data/scripts/dbbackup-docker-compose/dbbackup-docker-compose -c /home/data/scripts/dbbackup-docker-compose/default-docker-compose.conf --yearly +30 20 1 1 * root /home/data/scripts/docker-compose-dbbackup/dbbackup-docker-compose -c /home/data/scripts/docker-compose-dbbackup/default-docker-compose.conf --yearly diff --git a/dbbackup-docker-compose b/dbbackup-docker-compose index 0b8bcc3..afef4a4 100755 --- a/dbbackup-docker-compose +++ b/dbbackup-docker-compose @@ -29,7 +29,7 @@ VERBOSE=0 # # Stuff that should be universal for this install... # where we can find this app... -MAIN_DIR=/home/data/scripts/dbbackup-docker-compose +MAIN_DIR=/home/data/scripts/docker-compose-dbbackup # determine today's date DATE=`date '+%Y-%m-%d-%a'` # determine today's date From eb625966b0b9c1e8ab19db71d3d67f8f4da642db Mon Sep 17 00:00:00 2001 From: Dave Lane Date: Tue, 15 Nov 2022 06:47:53 +0000 Subject: [PATCH 09/11] Fixed path for config file in cron invocation. --- dbbackup-cron | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dbbackup-cron b/dbbackup-cron index 680824b..0afc49a 100644 --- a/dbbackup-cron +++ b/dbbackup-cron @@ -4,17 +4,17 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin # run system backups # # hourly - at 5 minutes past mon-sat -25 * * * * root /home/data/scripts/docker-compose-dbbackup/dbbackup-docker-compose -c /home/data/scripts/docker-compose-dbbackup/default-docker-compose.conf --hourly +25 * * * * root /home/data/scripts/docker-compose-dbbackup/dbbackup-docker-compose -c default-docker-compose.conf --hourly # # daily - at 7:30 pm, mon-sat -30 19 * * * root /home/data/scripts/docker-compose-dbbackup/dbbackup-docker-compose -c /home/data/scripts/docker-compose-dbbackup/default-docker-compose.conf --daily +30 19 * * * root /home/data/scripts/docker-compose-dbbackup/dbbackup-docker-compose -c default-docker-compose.conf --daily # # weekly - at 7:30 pm, sun -30 19 * * sun root /home/data/scripts/docker-compose-dbbackup/dbbackup-docker-compose -c /home/data/scripts/docker-compose-dbbackup/default-docker-compose.conf --weekly +30 19 * * sun root /home/data/scripts/docker-compose-dbbackup/dbbackup-docker-compose -c default-docker-compose.conf --weekly # # monthly - at 8:30 pm, on the first of the last of the month -30 20 1 * * root /home/data/scripts/docker-compose-dbbackup/dbbackup-docker-compose -c /home/data/scripts/docker-compose-dbbackup/default-docker-compose.conf --monthly +30 20 1 * * root /home/data/scripts/docker-compose-dbbackup/dbbackup-docker-compose -c default-docker-compose.conf --monthly # # yearly - at 8:30 pm, on the first of January. -30 20 1 1 * root /home/data/scripts/docker-compose-dbbackup/dbbackup-docker-compose -c /home/data/scripts/docker-compose-dbbackup/default-docker-compose.conf --yearly +30 20 1 1 * root /home/data/scripts/docker-compose-dbbackup/dbbackup-docker-compose -c default-docker-compose.conf --yearly From 35f979e8fafe184dc4cf3ea2fd778d1fd241e09a Mon Sep 17 00:00:00 2001 From: Dave Lane Date: Thu, 24 Nov 2022 03:23:20 +0000 Subject: [PATCH 10/11] Reverse update to reflect the new docker with integrated 'compose' and return to 'docker-compose' as the binary name. --- dbbackup-docker-compose | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbbackup-docker-compose b/dbbackup-docker-compose index afef4a4..7dcf62a 100755 --- a/dbbackup-docker-compose +++ b/dbbackup-docker-compose @@ -47,7 +47,7 @@ GREP=`which grep` # email program MAIL=`which mail` # database dump utility -DC=`which docker` +DC=`which docker-compose` #DEF_ARGS="-C -d -O -x" # # pattern for "ls" command to build list of @@ -136,13 +136,13 @@ delete_old() { do_backup() { FILE=$1 CONTAINER=$2 - VER=`$DC compose exec $CONTAINER pg_config --version` + VER=`$DC exec $CONTAINER pg_config --version` verbose "Postgres version $VER - echoing to backup: $FILE" echo "--" > $FILE echo "-- PostgreSQL Version: $VER" >> $FILE echo "--" >> $FILE echo "" >> $FILE - CMD="$DC compose exec $CONTAINER $DUMP_CMD" + CMD="$DC exec $CONTAINER $DUMP_CMD" verbose "doing database dump: $CMD" $CMD >> $FILE } From 515337682835baea25565b53d1efc1a9eff9f2f2 Mon Sep 17 00:00:00 2001 From: Dave Lane Date: Fri, 25 Nov 2022 16:17:05 +1300 Subject: [PATCH 11/11] adding NICE and NICE_VAL and ability to override backup numbers in config --- dbbackup-docker-compose | 10 +++++++--- default-docker-compose.conf-sample | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/dbbackup-docker-compose b/dbbackup-docker-compose index 7dcf62a..5e08f22 100755 --- a/dbbackup-docker-compose +++ b/dbbackup-docker-compose @@ -22,9 +22,11 @@ BU_FROOT_WEEKLY=weekly BU_FROOT_MONTHLY=monthly BU_FROOT_YEARLY=yearly # +# Set backup priority: -20 highest, 0 default, 19 lowest +NICE_VAL=0 # this can be overridden at invocation BU_CONF=default-docker-compose.conf -# # output for debugging... +# output for debugging... 0 = false, 1 = true VERBOSE=0 # # Stuff that should be universal for this install... @@ -49,6 +51,8 @@ MAIL=`which mail` # database dump utility DC=`which docker-compose` #DEF_ARGS="-C -d -O -x" +# nice +NICE=`which nice` # # pattern for "ls" command to build list of # pruneable backup files... @@ -142,7 +146,7 @@ do_backup() { echo "-- PostgreSQL Version: $VER" >> $FILE echo "--" >> $FILE echo "" >> $FILE - CMD="$DC exec $CONTAINER $DUMP_CMD" + CMD="$NICE -${NICE_VAL} $DC exec $CONTAINER $DUMP_CMD" verbose "doing database dump: $CMD" $CMD >> $FILE } @@ -230,7 +234,7 @@ do_backup $FILEPATH $DC_CONTAINER message "completed backup" # compress the backup message "compressing $FILEPATH" -$GZIP $FILEPATH +$NICE -${NICE_VAL} $GZIP $FILEPATH # return to where you started from... cd $OLD_DIR # diff --git a/default-docker-compose.conf-sample b/default-docker-compose.conf-sample index 0e4c8d2..6776d5d 100644 --- a/default-docker-compose.conf-sample +++ b/default-docker-compose.conf-sample @@ -17,3 +17,18 @@ DUMP_CMD='pg_dumpall -c -U postgres' # email address to send reports to, and subject EMAIL=webmaster@oerfoundation.org EMAIL_SUBJ="Mastodon on OERu Open Postgres Backup Report" +# +# Optional overrides +# +# Default retention +#BU_TO_KEEP_HOURLY=24 +#BU_TO_KEEP_DAILY=7 +#BU_TO_KEEP_WEEKLY=4 +#BU_TO_KEEP_MONTHLY=12 +#BU_TO_KEEP_YEARLY=7 +# +# Set backup priority: -20 highest, 0 default, 19 lowest +#NICE_VAL=0 +# +# Verbosity 0 = false, 1 = true +#VERBOSE=0