From f7a1ec367e5d14ce2f95a12c4414f208e83f79c3 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Sun, 28 Jun 2020 22:23:43 -0500 Subject: [PATCH 01/23] Building out smart sync --- gravity-sync.sh | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 17ef29d..bd5562c 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -367,6 +367,15 @@ function push_gs { } +function smart_gs { + md5_compare + + echo -e $primaryDBMD5 + echo -e $secondDBMD5 + echo -e $primaryCLMD5 + echo -e $secondCLMD5 +} + function restore_gs { MESSAGE="This will restore ${GRAVITY_FI} on $HOSTNAME with the previous version!" echo_warn @@ -1383,7 +1392,17 @@ function root_check { case $# in 0) - task_invalid + TASKTYPE='SYNC' + MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" + echo_good + + import_gs + validate_gs_folders + validate_ph_folders + validate_os_sshpass + + smart_gs + exit ;; 1) From 75eb9c769509c5e42b0020c3999b3902e9f73995 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 08:47:09 -0500 Subject: [PATCH 02/23] Write hashes from log out --- gravity-sync.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/gravity-sync.sh b/gravity-sync.sh index bd5562c..4388842 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -500,6 +500,15 @@ function logs_export { echo_stat echo -e $(date) "[${TASKTYPE}]" >> ${LOG_PATH}/${SYNCING_LOG} error_validate + + MESSAGE="Saving File Hashes" + echo_stat + rm -f backup/gravity-sync.md5 + echo -e ${primaryDBMD5} >> backup/gravity-sync.md5 + echo -e ${secondDBMD5} >> backup/gravity-sync.md5 + echo -e ${primaryCLMD5} >> backup/gravity-sync.md5 + echo -e ${secondCLMD5} >> backup/gravity-sync.md5 + error_validate } ### Output Sync Logs From 37fd78858db6926f6ccec465d2e84d1670b574a5 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 09:21:50 -0500 Subject: [PATCH 03/23] Import last hashes --- .gitignore | 1 + gravity-sync.sh | 28 +++++++++++++++++++--------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 410ed35..4a4bd19 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ gravity-sync.log gravity-sync.cron gravity.db.last gravity-sync.conf +gravity-sync.md5 backup/*.last backup/*.backup backup/*.push diff --git a/gravity-sync.sh b/gravity-sync.sh index 4388842..efd7d2e 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -370,10 +370,20 @@ function push_gs { function smart_gs { md5_compare + last_primaryDBMD5=$(sed "1q;d" gravity-sync.md5) + last_secondDBMD5=$(sed "2q;d" gravity-sync.md5) + last_primaryCIMD5=$(sed "3q;d" gravity-sync.md5) + last_secondCIMD5=$(sed "4q;d" gravity-sync.md5) + echo -e $primaryDBMD5 echo -e $secondDBMD5 echo -e $primaryCLMD5 echo -e $secondCLMD5 + + echo -e $last_primaryDBMD5 + echo -e $last_secondDBMD5 + echo -e $last_primaryCLMD5 + echo -e $last_secondCLMD5 } function restore_gs { @@ -496,19 +506,19 @@ function restore_gs { ## Core Logging ### Write Logs Out function logs_export { + MESSAGE="Saving File Hashes" + echo_stat + rm -f gravity-sync.md5 + echo -e ${primaryDBMD5} >> gravity-sync.md5 + echo -e ${secondDBMD5} >> gravity-sync.md5 + echo -e ${primaryCLMD5} >> gravity-sync.md5 + echo -e ${secondCLMD5} >> gravity-sync.md5 + error_validate + MESSAGE="Logging Successful ${TASKTYPE}" echo_stat echo -e $(date) "[${TASKTYPE}]" >> ${LOG_PATH}/${SYNCING_LOG} error_validate - - MESSAGE="Saving File Hashes" - echo_stat - rm -f backup/gravity-sync.md5 - echo -e ${primaryDBMD5} >> backup/gravity-sync.md5 - echo -e ${secondDBMD5} >> backup/gravity-sync.md5 - echo -e ${primaryCLMD5} >> backup/gravity-sync.md5 - echo -e ${secondCLMD5} >> backup/gravity-sync.md5 - error_validate } ### Output Sync Logs From 339d5a456f866aedce83c4530511802d1b28f397 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 09:25:14 -0500 Subject: [PATCH 04/23] Mark hashes --- gravity-sync.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index efd7d2e..5ec7278 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -375,15 +375,15 @@ function smart_gs { last_primaryCIMD5=$(sed "3q;d" gravity-sync.md5) last_secondCIMD5=$(sed "4q;d" gravity-sync.md5) - echo -e $primaryDBMD5 - echo -e $secondDBMD5 - echo -e $primaryCLMD5 - echo -e $secondCLMD5 + echo -e $primaryDBMD5 " 1" + echo -e $secondDBMD5 " 2" + echo -e $primaryCLMD5 " 3" + echo -e $secondCLMD5 " 4" - echo -e $last_primaryDBMD5 - echo -e $last_secondDBMD5 - echo -e $last_primaryCLMD5 - echo -e $last_secondCLMD5 + echo -e $last_primaryDBMD5 " 5" + echo -e $last_secondDBMD5 " 6" + echo -e $last_primaryCLMD5 " 7" + echo -e $last_secondCLMD5 " 8" } function restore_gs { From 06338d6a778ac4171242f73eafd56c56ab2f1397 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 12:57:13 -0500 Subject: [PATCH 05/23] Comparison logic --- gravity-sync.sh | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 5ec7278..1282f3f 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -372,18 +372,34 @@ function smart_gs { last_primaryDBMD5=$(sed "1q;d" gravity-sync.md5) last_secondDBMD5=$(sed "2q;d" gravity-sync.md5) - last_primaryCIMD5=$(sed "3q;d" gravity-sync.md5) - last_secondCIMD5=$(sed "4q;d" gravity-sync.md5) + last_primaryCLMD5=$(sed "3q;d" gravity-sync.md5) + last_secondCLMD5=$(sed "4q;d" gravity-sync.md5) - echo -e $primaryDBMD5 " 1" - echo -e $secondDBMD5 " 2" - echo -e $primaryCLMD5 " 3" - echo -e $secondCLMD5 " 4" + echo -e ${primaryDBMD5} " 1" + echo -e ${secondDBMD5} " 2" + echo -e ${primaryCLMD5} " 3" + echo -e ${secondCLMD5} " 4" - echo -e $last_primaryDBMD5 " 5" - echo -e $last_secondDBMD5 " 6" - echo -e $last_primaryCLMD5 " 7" - echo -e $last_secondCLMD5 " 8" + echo -e ${last_primaryDBMD5} " 5" + echo -e ${last_secondDBMD5} " 6" + echo -e ${last_primaryCLMD5} " 7" + echo -e ${last_secondCLMD5} " 8" + + if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] + then + echo "Primary DB has changed" + elif [ "${primaryDBMD5}" == "${last_primaryDBMD5}" ] + then + echo "Secondary DB has changed" + fi + + if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] + then + echo "Primary CL has changed" + elif [ "${primaryCLMD5}" == "${last_primaryCLMD5}" ] + then + echo "Secondary CL has changed" + fi } function restore_gs { From c229d5e12ebffe674381916d4f6dcd848e707b8a Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 13:04:38 -0500 Subject: [PATCH 06/23] No changes detected --- gravity-sync.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 1282f3f..dc760be 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -388,17 +388,21 @@ function smart_gs { if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] then echo "Primary DB has changed" - elif [ "${primaryDBMD5}" == "${last_primaryDBMD5}" ] + elif [ "${secondDBMD5}" != "${last_secondDBMD5}" ] then echo "Secondary DB has changed" + else + echo "No DB changes" fi if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] then echo "Primary CL has changed" - elif [ "${primaryCLMD5}" == "${last_primaryCLMD5}" ] + elif [ "${secondCLMD5}" != "${last_secondCLMD5}" ] then echo "Secondary CL has changed" + else + echo "No CL changes" fi } From afc33c54290f6bdce90460f14d97f8d68dfc5fe2 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 13:41:54 -0500 Subject: [PATCH 07/23] Break up push and pull into more subfunctions --- gravity-sync.sh | 101 ++++++++++++++++++++++++------------------------ 1 file changed, 50 insertions(+), 51 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index dc760be..53e8de8 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -135,10 +135,8 @@ function update_gs { } # Gravity Core Functions -## Pull Function -function pull_gs { - md5_compare - +## Pull Gravity +function pull_gs_grav { MESSAGE="Backing Up ${GRAVITY_FI} on $HOSTNAME" echo_stat cp ${PIHOLE_DIR}/${GRAVITY_FI} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${GRAVITY_FI}.backup >/dev/null 2>&1 @@ -158,45 +156,10 @@ function pull_gs { MESSAGE="Validating Settings of ${GRAVITY_FI}" echo_stat - - GRAVDB_OWN=$(ls -ld ${PIHOLE_DIR}/${GRAVITY_FI} | awk '{print $3 $4}') - if [ "$GRAVDB_OWN" != "piholepihole" ] - then - MESSAGE="Validating Ownership on ${GRAVITY_FI}" - echo_fail - - MESSAGE="Attempting to Compensate" - echo_warn - - MESSAGE="Setting Ownership on ${GRAVITY_FI}" - echo_stat - sudo chown pihole:pihole ${PIHOLE_DIR}/${GRAVITY_FI} >/dev/null 2>&1 - error_validate - - MESSAGE="Continuing Validation of ${GRAVITY_FI}" - echo_stat - fi - - GRAVDB_RWE=$(namei -m ${PIHOLE_DIR}/${GRAVITY_FI} | grep -v f: | grep ${GRAVITY_FI} | awk '{print $1}') - if [ "$GRAVDB_RWE" != "-rw-rw-r--" ] - then - MESSAGE="Validating Permissions on ${GRAVITY_FI}" - echo_fail - - MESSAGE="Attempting to Compensate" - echo_warn - - MESSAGE="Setting Permissions on ${GRAVITY_FI}" - echo_stat - sudo chmod 664 ${PIHOLE_DIR}/${GRAVITY_FI} >/dev/null 2>&1 - error_validate - - MESSAGE="Continuing Validation of ${GRAVITY_FI}" - echo_stat - fi - - echo_good +} +## Pull Custom +function pull_gs_cust { if [ "$SKIP_CUSTOM" != '1' ] then if [ "$REMOTE_CUSTOM_DNS" == "1" ] @@ -263,7 +226,10 @@ function pull_gs { echo_good fi fi +} +## Pull Reload +function pull_gs_reload { MESSAGE="Isolating Regeneration Pathways" echo_info sleep 1 @@ -277,17 +243,22 @@ function pull_gs { echo_stat ${PIHOLE_BIN} restartdns >/dev/null 2>&1 error_validate +} + +## Pull Function +function pull_gs { + md5_compare + + pull_gs_grav + pull_gs_cust + pull_gs_reload logs_export exit_withchange } -## Push Function -function push_gs { - md5_compare - - intent_validate - +## Push Gravity +function push_gs_grav { MESSAGE="Backing Up ${GRAVITY_FI} from ${REMOTE_HOST}" echo_stat RSYNC_REPATH="rsync" @@ -313,7 +284,10 @@ function push_gs { CMD_TIMEOUT='15' CMD_REQUESTED="sudo chown pihole:pihole ${PIHOLE_DIR}/${GRAVITY_FI}" create_sshcmd +} +## Push Custom +function push_gs_cust { if [ "$SKIP_CUSTOM" != '1' ] then if [ "$REMOTE_CUSTOM_DNS" == "1" ] @@ -345,28 +319,42 @@ function push_gs { create_sshcmd fi fi +} +## Push Reload +function push_gs_reload { MESSAGE="Inverting Tachyon Pulses" echo_info sleep 1 - MESSAGE="Updating FTLDNS Configuration" + MESSAGE="Updating Remote FTLDNS Configuration" echo_stat CMD_TIMEOUT='15' CMD_REQUESTED="${PIHOLE_BIN} restartdns reloadlists" create_sshcmd - MESSAGE="Reloading FTLDNS Services" + MESSAGE="Reloading Remote FTLDNS Services" echo_stat CMD_TIMEOUT='15' CMD_REQUESTED="${PIHOLE_BIN} restartdns" create_sshcmd +} + +## Push Function +function push_gs { + md5_compare + + intent_validate + + push_gs_grav + push_gs_cust + push_gs_reload logs_export exit_withchange - } +## Smart Sync Function function smart_gs { md5_compare @@ -385,12 +373,19 @@ function smart_gs { echo -e ${last_primaryCLMD5} " 7" echo -e ${last_secondCLMD5} " 8" + DB_SMARTPULL="0" + DB_SMARTPUSH="0" + CL_SMARTPULL="0" + CL_SMARTPUSH="0" + if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] then echo "Primary DB has changed" + DB_SMARTPULL="1" elif [ "${secondDBMD5}" != "${last_secondDBMD5}" ] then echo "Secondary DB has changed" + DB_SMARTPUSH="1" else echo "No DB changes" fi @@ -398,12 +393,16 @@ function smart_gs { if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] then echo "Primary CL has changed" + CL_SMARTPULL="1" elif [ "${secondCLMD5}" != "${last_secondCLMD5}" ] then echo "Secondary CL has changed" + CL_SMARTPUSH="1" else echo "No CL changes" fi + + } function restore_gs { From 570d8fc86d7fbf32ff5e9fcd2f8ebe59c86ad5ab Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 13:48:54 -0500 Subject: [PATCH 08/23] Oops --- gravity-sync.sh | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/gravity-sync.sh b/gravity-sync.sh index 53e8de8..494f479 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -156,6 +156,44 @@ function pull_gs_grav { MESSAGE="Validating Settings of ${GRAVITY_FI}" echo_stat + + GRAVDB_OWN=$(ls -ld ${PIHOLE_DIR}/${GRAVITY_FI} | awk '{print $3 $4}') + if [ "$GRAVDB_OWN" != "piholepihole" ] + then + MESSAGE="Validating Ownership on ${GRAVITY_FI}" + echo_fail + + MESSAGE="Attempting to Compensate" + echo_warn + + MESSAGE="Setting Ownership on ${GRAVITY_FI}" + echo_stat + sudo chown pihole:pihole ${PIHOLE_DIR}/${GRAVITY_FI} >/dev/null 2>&1 + error_validate + + MESSAGE="Continuing Validation of ${GRAVITY_FI}" + echo_stat + fi + + GRAVDB_RWE=$(namei -m ${PIHOLE_DIR}/${GRAVITY_FI} | grep -v f: | grep ${GRAVITY_FI} | awk '{print $1}') + if [ "$GRAVDB_RWE" != "-rw-rw-r--" ] + then + MESSAGE="Validating Permissions on ${GRAVITY_FI}" + echo_fail + + MESSAGE="Attempting to Compensate" + echo_warn + + MESSAGE="Setting Permissions on ${GRAVITY_FI}" + echo_stat + sudo chmod 664 ${PIHOLE_DIR}/${GRAVITY_FI} >/dev/null 2>&1 + error_validate + + MESSAGE="Continuing Validation of ${GRAVITY_FI}" + echo_stat + fi + + echo_good } ## Pull Custom From 9abf85c3f7af1495e973cc525447f2046a82995a Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 14:03:47 -0500 Subject: [PATCH 09/23] HISTORY_MD5 --- ADVANCED.md | 4 ++-- VERSION | 2 +- gravity-sync.sh | 11 ++++++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index 62dd9cd..e3f3ea8 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -14,8 +14,8 @@ Download the latest release from [GitHub](https://github.com/vmstan/gravity-sync ```bash cd ~ -wget https://github.com/vmstan/gravity-sync/archive/v1.8.3.zip -unzip v1.8.3.zip -d gravity-sync +wget https://github.com/vmstan/gravity-sync/archive/v2.0.0.zip +unzip v2.0.0.zip -d gravity-sync cd gravity-sync ``` diff --git a/VERSION b/VERSION index fe4e75f..359a5b9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.8.3 \ No newline at end of file +2.0.0 \ No newline at end of file diff --git a/gravity-sync.sh b/gravity-sync.sh index 494f479..bece31b 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -3,7 +3,7 @@ SCRIPT_START=$SECONDS # GRAVITY SYNC BY VMSTAN ##################### PROGRAM='Gravity Sync' -VERSION='1.8.3' +VERSION='2.0.0' # Execute from the home folder of the user who owns it (ex: 'cd ~/gravity-sync') # For documentation or downloading updates visit https://github.com/vmstan/gravity-sync @@ -24,6 +24,7 @@ BACKUP_FOLD='backup' # must exist as subdirectory in LOCAL_FOLDR LOG_PATH="$HOME/${LOCAL_FOLDR}" # replace in gravity-sync.conf to overwrite SYNCING_LOG='gravity-sync.log' # replace in gravity-sync.conf to overwrite CRONJOB_LOG='gravity-sync.cron' # replace in gravity-sync.conf to overwrite +HISTORY_MD5='gravity-sync.md5' # replace in gravity-sync.conf to overwrite # Interaction Customization VERIFY_PASS='0' # replace in gravity-sync.conf to overwrite @@ -396,10 +397,10 @@ function push_gs { function smart_gs { md5_compare - last_primaryDBMD5=$(sed "1q;d" gravity-sync.md5) - last_secondDBMD5=$(sed "2q;d" gravity-sync.md5) - last_primaryCLMD5=$(sed "3q;d" gravity-sync.md5) - last_secondCLMD5=$(sed "4q;d" gravity-sync.md5) + last_primaryDBMD5=$(sed "1q;d" ${HISTORY_MD5}) + last_secondDBMD5=$(sed "2q;d" ${HISTORY_MD5}) + last_primaryCLMD5=$(sed "3q;d" ${HISTORY_MD5}) + last_secondCLMD5=$(sed "4q;d" ${HISTORY_MD5}) echo -e ${primaryDBMD5} " 1" echo -e ${secondDBMD5} " 2" From 5cee345ee938ca51255d1eadbae41bbe711f045a Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 14:42:53 -0500 Subject: [PATCH 10/23] Automate update --- CHANGELOG.md | 12 ++++++++++++ gravity-sync.sh | 31 ++++++------------------------- 2 files changed, 18 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4db75da..75ad9f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # The Changelog +## 2.0 +### The Smart Release + +**Features** +For 2.0, the `push` and `pull` functions have been broken up for each component part (`gravity.db` and `custom.list`), and Gravity Sync will now detect not only if each component has changed since the last sync but also what direction they need to go. + +Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull and a push of each part and then restart the correct components on each server. + +This allows you to be more flexible in where you make your configuration changes to block/allow lists, but it's still suggested to identify a primary server to make these changes on. In the event there are configuration changes to the same element (`custom.list` changes at both sides) then the primary will still be the authoritative source and overwrite the secondary copy. + +Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. + ## 1.8 ### The Logical Release diff --git a/gravity-sync.sh b/gravity-sync.sh index bece31b..73fbfed 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -402,41 +402,22 @@ function smart_gs { last_primaryCLMD5=$(sed "3q;d" ${HISTORY_MD5}) last_secondCLMD5=$(sed "4q;d" ${HISTORY_MD5}) - echo -e ${primaryDBMD5} " 1" - echo -e ${secondDBMD5} " 2" - echo -e ${primaryCLMD5} " 3" - echo -e ${secondCLMD5} " 4" - - echo -e ${last_primaryDBMD5} " 5" - echo -e ${last_secondDBMD5} " 6" - echo -e ${last_primaryCLMD5} " 7" - echo -e ${last_secondCLMD5} " 8" - - DB_SMARTPULL="0" - DB_SMARTPUSH="0" - CL_SMARTPULL="0" - CL_SMARTPUSH="0" - if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] then - echo "Primary DB has changed" - DB_SMARTPULL="1" + pull_gs_grav elif [ "${secondDBMD5}" != "${last_secondDBMD5}" ] then - echo "Secondary DB has changed" - DB_SMARTPUSH="1" + push_gs_grav else echo "No DB changes" fi if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] then - echo "Primary CL has changed" - CL_SMARTPULL="1" + pull_gs_cust elif [ "${secondCLMD5}" != "${last_secondCLMD5}" ] then - echo "Secondary CL has changed" - CL_SMARTPUSH="1" + push_gs_cust else echo "No CL changes" fi @@ -1293,7 +1274,7 @@ function task_automate { MESSAGE="Saving New Automation" echo_stat - (crontab -l 2>/dev/null; echo "*/${INPUT_AUTO_FREQ} * * * * ${BASH_PATH} $HOME/${LOCAL_FOLDR}/${GS_FILENAME} pull > ${LOG_PATH}/${CRONJOB_LOG}") | crontab - + (crontab -l 2>/dev/null; echo "*/${INPUT_AUTO_FREQ} * * * * ${BASH_PATH} $HOME/${LOCAL_FOLDR}/${GS_FILENAME} > ${LOG_PATH}/${CRONJOB_LOG}") | crontab - error_validate fi exit_withchange @@ -1305,7 +1286,7 @@ function clear_cron { echo_stat crontab -l > cronjob-old.tmp - sed '/.sh pull/d' cronjob-old.tmp > cronjob-new.tmp + sed '/${GS_FILENAME}/d' cronjob-old.tmp > cronjob-new.tmp crontab cronjob-new.tmp 2>/dev/null error_validate rm cronjob-old.tmp From adb63debc13f0cd8d1d857c59699b0d924323291 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 15:49:16 -0500 Subject: [PATCH 11/23] Full sync logic --- CHANGELOG.md | 6 +++--- gravity-sync.sh | 40 +++++++++++++++++++++++++++++++++++----- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75ad9f9..c17a8e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,11 @@ **Features** For 2.0, the `push` and `pull` functions have been broken up for each component part (`gravity.db` and `custom.list`), and Gravity Sync will now detect not only if each component has changed since the last sync but also what direction they need to go. -Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull and a push of each part and then restart the correct components on each server. +Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull and then push for each part, and then restart the correct components on each server. It will also only perform a sync of each component if there are changes within each type to replicate. -This allows you to be more flexible in where you make your configuration changes to block/allow lists, but it's still suggested to identify a primary server to make these changes on. In the event there are configuration changes to the same element (`custom.list` changes at both sides) then the primary will still be the authoritative source and overwrite the secondary copy. +This allows you to be more flexible in where you make your configuration changes to block/allow lists and local DNS settings being made on either the primary or secondary, but it's best practice to continue making changes on the primary server where possible. In the event there are configuration changes to the same element (`custom.list` changes at both sides) then the primary will still be the authoritative source and overwrite the secondary copy. -Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. +Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. You can also continue to use `pull` in your automation if you choose. New installs will use the smart Sync function by default. Running `./gravity-sync.sh` without any arguments will now run the smart Sync function. ## 1.8 ### The Logical Release diff --git a/gravity-sync.sh b/gravity-sync.sh index 73fbfed..be26c32 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -405,24 +405,39 @@ function smart_gs { if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] then pull_gs_grav + PULLRESTART="1" elif [ "${secondDBMD5}" != "${last_secondDBMD5}" ] then push_gs_grav + PUSHRESTART="1" else - echo "No DB changes" + # echo "No DB changes" fi if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] then pull_gs_cust + PULLRESTART="1" elif [ "${secondCLMD5}" != "${last_secondCLMD5}" ] then push_gs_cust + PUSHRESTART="1" else - echo "No CL changes" + # echo "No CL changes" fi + if [ "$PULLRESTART" == "1" ] + then + pull_gs_reload + fi + if [ "$PUSHRESTART" == "1" ] + then + push_gs_reload + fi + + logs_export + exit_withchange } function restore_gs { @@ -1178,9 +1193,10 @@ function list_gs_arguments { echo -e " ${YELLOW}version${NC} Display installed version of ${PROGRAM}" echo -e "" echo -e "Replication Options:" - echo -e " ${YELLOW}pull${NC} Sync remote ${GRAVITY_FI} to this server" - echo -e " ${YELLOW}push${NC} Force changes made on this server back" - echo -e " ${YELLOW}restore${NC} Restore ${GRAVITY_FI} on this server" + echo -e " ${YELLOW}sync${NC} Detect changes on each side and bring them together" + echo -e " ${YELLOW}pull${NC} Force remote configuration changes to this server" + echo -e " ${YELLOW}push${NC} Force local configuration made on this server back" + echo -e " ${YELLOW}restore${NC} Restore the ${GRAVITY_FI} on this server" echo -e " ${YELLOW}compare${NC} Just check for differences" echo -e "" echo -e "Debug Options:" @@ -1465,6 +1481,20 @@ case $# in 1) case $1 in + sync) + TASKTYPE='SYNC' + MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" + echo_good + + import_gs + validate_gs_folders + validate_ph_folders + validate_os_sshpass + + smart_gs + exit + ;; + pull) TASKTYPE='PULL' MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" From fa270aff677033b9c749dd36fde53b2fee76cd60 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 15:50:58 -0500 Subject: [PATCH 12/23] Validate changes --- gravity-sync.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gravity-sync.sh b/gravity-sync.sh index be26c32..31e768d 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -436,6 +436,8 @@ function smart_gs { push_gs_reload fi + md5_compare + logs_export exit_withchange } From 0e72da570f4da25717d393b137c3cbee821ade08 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 16:06:28 -0500 Subject: [PATCH 13/23] Recompare --- gravity-sync.sh | 84 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 4 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 31e768d..b0bafd6 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -410,8 +410,6 @@ function smart_gs { then push_gs_grav PUSHRESTART="1" - else - # echo "No DB changes" fi if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] @@ -422,8 +420,6 @@ function smart_gs { then push_gs_cust PUSHRESTART="1" - else - # echo "No CL changes" fi if [ "$PULLRESTART" == "1" ] @@ -1019,6 +1015,86 @@ function md5_compare { fi } +function md5_recheck { + MESSAGE="Performing Replicator Diagnostics" + echo_info + + HASHMARK='0' + + MESSAGE="Reanalyzing ${GRAVITY_FI} on ${REMOTE_HOST}" + echo_stat + primaryDBMD5=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "md5sum ${PIHOLE_DIR}/${GRAVITY_FI}" | sed 's/\s.*$//') + error_validate + + MESSAGE="Reanalyzing ${GRAVITY_FI} on $HOSTNAME" + echo_stat + secondDBMD5=$(md5sum ${PIHOLE_DIR}/${GRAVITY_FI} | sed 's/\s.*$//') + error_validate + + if [ "$primaryDBMD5" == "$secondDBMD5" ] + then + HASHMARK=$((HASHMARK+0)) + else + MESSAGE="Differenced ${GRAVITY_FI} Detected" + echo_warn + HASHMARK=$((HASHMARK+1)) + fi + + if [ "$SKIP_CUSTOM" != '1' ] + then + if [ -f ${PIHOLE_DIR}/${CUSTOM_DNS} ] + then + if ${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} test -e ${PIHOLE_DIR}/${CUSTOM_DNS} + then + REMOTE_CUSTOM_DNS="1" + MESSAGE="Reanalyzing ${CUSTOM_DNS} on ${REMOTE_HOST}" + echo_stat + + primaryCLMD5=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "md5sum ${PIHOLE_DIR}/${CUSTOM_DNS} | sed 's/\s.*$//'") + error_validate + + MESSAGE="Reanalyzing ${CUSTOM_DNS} on $HOSTNAME" + echo_stat + secondCLMD5=$(md5sum ${PIHOLE_DIR}/${CUSTOM_DNS} | sed 's/\s.*$//') + error_validate + + if [ "$primaryCLMD5" == "$secondCLMD5" ] + then + # MESSAGE="${CUSTOM_DNS} Identical" + # echo_info + HASHMARK=$((HASHMARK+0)) + else + MESSAGE="Differenced ${CUSTOM_DNS} Detected" + echo_warn + HASHMARK=$((HASHMARK+1)) + fi + else + MESSAGE="No ${CUSTOM_DNS} Detected on ${REMOTE_HOST}" + echo_info + fi + else + if ${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} test -e ${PIHOLE_DIR}/${CUSTOM_DNS} + then + REMOTE_CUSTOM_DNS="1" + MESSAGE="${REMOTE_HOST} has ${CUSTOM_DNS}" + HASHMARK=$((HASHMARK+1)) + echo_info + fi + MESSAGE="No ${CUSTOM_DNS} Detected on $HOSTNAME" + echo_info + fi + fi + + if [ "$HASHMARK" != "0" ] + then + MESSAGE="Replication Checks Failed" + echo_warn + else + MESSAGE="Replication Was Successful" + echo_info + fi +} + ## Validate Intent function intent_validate { if [ "$VERIFY_PASS" == "0" ] From 3114f72e4c511ec1ab70e2d67a8504bde2f37c47 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 16:08:06 -0500 Subject: [PATCH 14/23] recheck --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index b0bafd6..c4ce2e6 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -432,7 +432,7 @@ function smart_gs { push_gs_reload fi - md5_compare + md5_recheckr logs_export exit_withchange From c0b594b2bc323347043e40dc2eaef24ee83a12ae Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Mon, 29 Jun 2020 16:08:27 -0500 Subject: [PATCH 15/23] No r --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index c4ce2e6..6ee523c 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -432,7 +432,7 @@ function smart_gs { push_gs_reload fi - md5_recheckr + md5_recheck logs_export exit_withchange From 1ef5fec67653ee53d140bb37729c9b64e4728e72 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 14:26:45 -0500 Subject: [PATCH 16/23] building both sides check --- gravity-sync.sh | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 6ee523c..d72e1b1 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -402,14 +402,40 @@ function smart_gs { last_primaryCLMD5=$(sed "3q;d" ${HISTORY_MD5}) last_secondCLMD5=$(sed "4q;d" ${HISTORY_MD5}) + PRIDBCHANGE="0" + SECDBCHANGE="0" + PRICLCHANGE="0" + SECICLCHANGE="0" + if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] then - pull_gs_grav - PULLRESTART="1" - elif [ "${secondDBMD5}" != "${last_secondDBMD5}" ] + PRIDBCHANGE="1" + # pull_gs_grav + # PULLRESTART="1" + fi + + if [ "${secondDBMD5}" != "${last_secondDBMD5}" ] then - push_gs_grav - PUSHRESTART="1" + SECDBCHANGE="1" + # push_gs_grav + # PUSHRESTART="1" + fi + + if [ "${PRIDBCHANGE}" == "${SECDBCHANGE}" ] + then + if [ "${PRIDBCHANGE}" != "0" ] + then + echo "Both sides have changed" + fi + else + then + if [ "${PRIDBCHANGE}" != "0" ] + then + echo "Primary has changed" + elif [ "${SECDBCHANGE}" != "0" ] + then + echo "Secondary has changed" + fi fi if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] From c604697d5455eaaa01435d546bb8768d85fd5bdc Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 14:29:38 -0500 Subject: [PATCH 17/23] logic insert --- gravity-sync.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index d72e1b1..a48a043 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -426,15 +426,17 @@ function smart_gs { if [ "${PRIDBCHANGE}" != "0" ] then echo "Both sides have changed" + exit fi else - then if [ "${PRIDBCHANGE}" != "0" ] then - echo "Primary has changed" + pull_gs_grav + PULLRESTART="1" elif [ "${SECDBCHANGE}" != "0" ] then - echo "Secondary has changed" + push_gs_grav + PUSHRESTART="1" fi fi From d3966191f01df8835ea8846eb0b39dc77c1b1b16 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 14:55:52 -0500 Subject: [PATCH 18/23] checks dates --- gravity-sync.sh | 73 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index a48a043..089e848 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -410,23 +410,37 @@ function smart_gs { if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] then PRIDBCHANGE="1" - # pull_gs_grav - # PULLRESTART="1" fi if [ "${secondDBMD5}" != "${last_secondDBMD5}" ] then SECDBCHANGE="1" - # push_gs_grav - # PUSHRESTART="1" fi if [ "${PRIDBCHANGE}" == "${SECDBCHANGE}" ] then if [ "${PRIDBCHANGE}" != "0" ] then - echo "Both sides have changed" - exit + MESSAGE="Both Sides Have Changed" + echo_warn + + PRIDBDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${GRAVITY_FI}") + SECDBDATE=$(stat -c %Y ${PIHOLE_DIR}/${GRAVITY_FI}) + + if [ "${PRIDBDATE}" -gt "$SECDBDATE" ] + then + MESSAGE="Primary was Last Changed" + echo_info + + pull_gs_grav + PULLRESTART="1" + else + MESSAGE="Secondary was Last Changed" + echo_info + + push_gs_grav + PUSHRESTART="1" + fi fi else if [ "${PRIDBCHANGE}" != "0" ] @@ -442,12 +456,49 @@ function smart_gs { if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ] then - pull_gs_cust - PULLRESTART="1" - elif [ "${secondCLMD5}" != "${last_secondCLMD5}" ] + PRICLCHANGE="1" + fi + + if [ "${secondCLMD5}" != "${last_secondCLMD5}" ] then - push_gs_cust - PUSHRESTART="1" + SECCLCHANGE="1" + fi + + if [ "${PRICLCHANGE}" == "${SECCLCHANGE}" ] + then + if [ "${PRICLCHANGE}" != "0" ] + then + MESSAGE="Both Sides Have Changed" + echo_warn + + PRICLDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${CUSTOM_DNS}") + SECCLDATE=$(stat -c %Y ${PIHOLE_DIR}/${CUSTOM_DNS}) + + if [ "${PRICLDATE}" -gt "$SECCLDATE" ] + then + MESSAGE="Primary was Last Changed" + echo_info + + pull_gs_cust + PULLRESTART="1" + else + MESSAGE="Secondary was Last Changed" + echo_info + + push_gs_cust + PUSHRESTART="1" + fi + fi + else + if [ "${PRICLCHANGE}" != "0" ] + then + pull_gs_cust + PULLRESTART="1" + elif [ "${SECCLCHANGE}" != "0" ] + then + push_gs_cust + PUSHRESTART="1" + fi fi if [ "$PULLRESTART" == "1" ] From c7a6ba4f692bacc7d60d5667fa64cbaee708f0a7 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 15:00:08 -0500 Subject: [PATCH 19/23] extra letter --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 089e848..a64bd6c 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -405,7 +405,7 @@ function smart_gs { PRIDBCHANGE="0" SECDBCHANGE="0" PRICLCHANGE="0" - SECICLCHANGE="0" + SECCLCHANGE="0" if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ] then From d5ededce123c916ff07f94e3483fd4f7a3e957e9 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 15:15:42 -0500 Subject: [PATCH 20/23] checks for history_md5 existing --- ADVANCED.md | 2 +- CHANGELOG.md | 2 +- gravity-sync.sh | 56 ++++++++++++++++++++++++++++++++++--------------- 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index e3f3ea8..c9c408a 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -173,5 +173,5 @@ If you prefer to still use cron but modify your settings by hand, using the entr ```bash crontab -e -*/30 * * * * /bin/bash /home/USER/gravity-sync/gravity-sync.sh pull > /home/USER/gravity-sync/gravity-sync.cron +*/30 * * * * /bin/bash /home/USER/gravity-sync/gravity-sync.sh > /home/USER/gravity-sync/gravity-sync.cron ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index c17a8e9..22197e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ For 2.0, the `push` and `pull` functions have been broken up for each component Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull and then push for each part, and then restart the correct components on each server. It will also only perform a sync of each component if there are changes within each type to replicate. -This allows you to be more flexible in where you make your configuration changes to block/allow lists and local DNS settings being made on either the primary or secondary, but it's best practice to continue making changes on the primary server where possible. In the event there are configuration changes to the same element (`custom.list` changes at both sides) then the primary will still be the authoritative source and overwrite the secondary copy. +This allows you to be more flexible in where you make your configuration changes to block/allow lists and local DNS settings being made on either the primary or secondary, but it's best practice to continue making changes on the primary server where possible. In the event there are configuration changes to the same element (`custom.list` changes at both sides) then Gravity Sync will attempt to determine based on timestamps on what side the last changed happened, in which case the latest changes will be considered authoritative and overwrite the other side. Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. You can also continue to use `pull` in your automation if you choose. New installs will use the smart Sync function by default. Running `./gravity-sync.sh` without any arguments will now run the smart Sync function. diff --git a/gravity-sync.sh b/gravity-sync.sh index a64bd6c..9f6096c 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -397,10 +397,18 @@ function push_gs { function smart_gs { md5_compare - last_primaryDBMD5=$(sed "1q;d" ${HISTORY_MD5}) - last_secondDBMD5=$(sed "2q;d" ${HISTORY_MD5}) - last_primaryCLMD5=$(sed "3q;d" ${HISTORY_MD5}) - last_secondCLMD5=$(sed "4q;d" ${HISTORY_MD5}) + if [ -f "${LOG_PATH}/${HISTORY_MD5}" ] + then + last_primaryDBMD5=$(sed "1q;d" ${LOG_PATH}/${HISTORY_MD5}) + last_secondDBMD5=$(sed "2q;d" ${LOG_PATH}/${HISTORY_MD5}) + last_primaryCLMD5=$(sed "3q;d" ${LOG_PATH}/${HISTORY_MD5}) + last_secondCLMD5=$(sed "4q;d" ${LOG_PATH}/{HISTORY_MD5}) + else + last_primaryDBMD5="0" + last_secondDBMD5="0" + last_primaryCLMD5="0" + last_secondCLMD5="0" + fi PRIDBCHANGE="0" SECDBCHANGE="0" @@ -421,7 +429,7 @@ function smart_gs { then if [ "${PRIDBCHANGE}" != "0" ] then - MESSAGE="Both Sides Have Changed" + MESSAGE="Both Sides Gravity Database Changed" echo_warn PRIDBDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${GRAVITY_FI}") @@ -429,13 +437,13 @@ function smart_gs { if [ "${PRIDBDATE}" -gt "$SECDBDATE" ] then - MESSAGE="Primary was Last Changed" + MESSAGE="Primary Gravity Database Last Changed" echo_info pull_gs_grav PULLRESTART="1" else - MESSAGE="Secondary was Last Changed" + MESSAGE="Secondary Gravity Database Last Changed" echo_info push_gs_grav @@ -468,7 +476,7 @@ function smart_gs { then if [ "${PRICLCHANGE}" != "0" ] then - MESSAGE="Both Sides Have Changed" + MESSAGE="Both Sides Custom DNS Changed" echo_warn PRICLDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${CUSTOM_DNS}") @@ -476,13 +484,13 @@ function smart_gs { if [ "${PRICLDATE}" -gt "$SECCLDATE" ] then - MESSAGE="Primary was Last Changed" + MESSAGE="Primary Custom DNS Last Changed" echo_info pull_gs_cust PULLRESTART="1" else - MESSAGE="Secondary was Last Changed" + MESSAGE="Secondary Custom DNS Last Changed" echo_info push_gs_cust @@ -639,11 +647,11 @@ function restore_gs { function logs_export { MESSAGE="Saving File Hashes" echo_stat - rm -f gravity-sync.md5 - echo -e ${primaryDBMD5} >> gravity-sync.md5 - echo -e ${secondDBMD5} >> gravity-sync.md5 - echo -e ${primaryCLMD5} >> gravity-sync.md5 - echo -e ${secondCLMD5} >> gravity-sync.md5 + rm -f ${LOG_PATH}/${HISTORY_MD5} + echo -e ${primaryDBMD5} >> ${LOG_PATH}/${HISTORY_MD5} + echo -e ${secondDBMD5} >> ${LOG_PATH}/${HISTORY_MD5} + echo -e ${primaryCLMD5} >> ${LOG_PATH}/${HISTORY_MD5} + echo -e ${secondCLMD5} >> ${LOG_PATH}/${HISTORY_MD5} error_validate MESSAGE="Logging Successful ${TASKTYPE}" @@ -1623,7 +1631,7 @@ function root_check { case $# in 0) - TASKTYPE='SYNC' + TASKTYPE='SMART' MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good @@ -1639,7 +1647,21 @@ case $# in 1) case $1 in sync) - TASKTYPE='SYNC' + TASKTYPE='SMART' + MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" + echo_good + + import_gs + validate_gs_folders + validate_ph_folders + validate_os_sshpass + + smart_gs + exit + ;; + + smart) + TASKTYPE='SMART' MESSAGE="${MESSAGE}: ${TASKTYPE} Requested" echo_good From 95dd3d56347d304cd5ee38bcc7035e06fe075510 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 15:19:59 -0500 Subject: [PATCH 21/23] missing $ --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index 9f6096c..a968bb6 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -402,7 +402,7 @@ function smart_gs { last_primaryDBMD5=$(sed "1q;d" ${LOG_PATH}/${HISTORY_MD5}) last_secondDBMD5=$(sed "2q;d" ${LOG_PATH}/${HISTORY_MD5}) last_primaryCLMD5=$(sed "3q;d" ${LOG_PATH}/${HISTORY_MD5}) - last_secondCLMD5=$(sed "4q;d" ${LOG_PATH}/{HISTORY_MD5}) + last_secondCLMD5=$(sed "4q;d" ${LOG_PATH}/{$HISTORY_MD5}) else last_primaryDBMD5="0" last_secondDBMD5="0" From 6fe6a467948360e3a811938e43d112d61fc403fc Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 15:21:58 -0500 Subject: [PATCH 22/23] BAH --- gravity-sync.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity-sync.sh b/gravity-sync.sh index a968bb6..c180e9b 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -402,7 +402,7 @@ function smart_gs { last_primaryDBMD5=$(sed "1q;d" ${LOG_PATH}/${HISTORY_MD5}) last_secondDBMD5=$(sed "2q;d" ${LOG_PATH}/${HISTORY_MD5}) last_primaryCLMD5=$(sed "3q;d" ${LOG_PATH}/${HISTORY_MD5}) - last_secondCLMD5=$(sed "4q;d" ${LOG_PATH}/{$HISTORY_MD5}) + last_secondCLMD5=$(sed "4q;d" ${LOG_PATH}/${HISTORY_MD5}) else last_primaryDBMD5="0" last_secondDBMD5="0" From 583c7c84005e5f9b3061c70f2ce08f3f92d910d3 Mon Sep 17 00:00:00 2001 From: Michael Stanclift Date: Tue, 7 Jul 2020 15:37:20 -0500 Subject: [PATCH 23/23] 2.0.0 --- ADVANCED.md | 7 ++++++- CHANGELOG.md | 10 ++++++---- gravity-sync.conf.example | 1 + gravity-sync.sh | 14 ++++++++------ 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/ADVANCED.md b/ADVANCED.md index c9c408a..92a825d 100644 --- a/ADVANCED.md +++ b/ADVANCED.md @@ -97,7 +97,7 @@ Gravity Sync will place logs in the same folder as the script (identified as .cr Default setting in Gravity Sync is `$HOME/${LOCAL_FOLDR}` #### `SYNCING_LOG=''` -Gravity Sync will write a timestamp for any completed pull, push or restore job to this file. If you want to change the name of this file, you will also need to adjust the LOG_PATH variable above, otherwise your file will be remove during an `update` operations. +Gravity Sync will write a timestamp for any completed sync, pull, push or restore job to this file. If you want to change the name of this file, you will also need to adjust the LOG_PATH variable above, otherwise your file will be remove during an `update` operations. Default setting in Gravity Sync is `gravity-sync.log` @@ -108,6 +108,11 @@ This will have an impact to both the `./gravity-sync.sh automate` function and t Default setting in Gravity Sync is `gravity-sync.cron` +#### `HISTORY_MD5=''` +Gravity Sync will log the file hashes of the previous `smart` task to this file. If you want to change the name of this file, you will also need to adjust the LOG_PATH variable above, otherwise your file will be removed during an `update` operations. + +Default setting in Gravity Sync is `gravity-sync.md5` + #### `VERIFY_PASS=''` Gravity Sync will prompt to verify user interactivity during push, restore, or config operations (that overwrite an existing configuration) with the intention that it prevents someone from accidentally automating in the wrong direction or overwriting data intentionally. If you'd like to automate a push function, or just don't like to be asked twice to do something distructive, then you can opt-out. diff --git a/CHANGELOG.md b/CHANGELOG.md index 22197e2..c57efdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,13 +4,15 @@ ### The Smart Release **Features** -For 2.0, the `push` and `pull` functions have been broken up for each component part (`gravity.db` and `custom.list`), and Gravity Sync will now detect not only if each component has changed since the last sync but also what direction they need to go. +In this release, Gravity Sync will now detect not only if each component (`gravity.db` and `custom.list`) has changed since the last sync, but also what direction they need to go. It will then initate a `push` and/or `pull` specific to each piece. -Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull and then push for each part, and then restart the correct components on each server. It will also only perform a sync of each component if there are changes within each type to replicate. +Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull of the `gravity.db` then push `custom.list` and finally restart the correct components on each server. It will also now only perform a sync of each component if there are changes within each type to replicate. So if you only make a small change to your Local DNS settings, it doesn't kickoff the larger `gravity.db` replication. -This allows you to be more flexible in where you make your configuration changes to block/allow lists and local DNS settings being made on either the primary or secondary, but it's best practice to continue making changes on the primary server where possible. In the event there are configuration changes to the same element (`custom.list` changes at both sides) then Gravity Sync will attempt to determine based on timestamps on what side the last changed happened, in which case the latest changes will be considered authoritative and overwrite the other side. +The default command for Gravity Sync is now just `./gravity-sync.sh` -- but you can also run `./gravity-sync.sh smart` if you feel like it, and it'll do the same thing. -Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. You can also continue to use `pull` in your automation if you choose. New installs will use the smart Sync function by default. Running `./gravity-sync.sh` without any arguments will now run the smart Sync function. +This allows you to be more flexible in where you make your configuration changes to block/allow lists and local DNS settings being made on either the primary or secondary, but it's best practice to continue making changes on one side where possible. In the event there are configuration changes to the same element (example, `custom.list` changes at both sides) then Gravity Sync will attempt to determine based on timestamps on what side the last changed happened, in which case the latest changes will be considered authoritative and overwrite the other side. Gravity Sync does not merge the contents of the files when changes happen, it simply overwrites the entire content. + +New installs will use the `smart` function by default. Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. This is not required. The previous `./gravity-sync.sh pull` and `./gravity-sync.sh push` commands continue to function as they did previously, with no intention to break this functionality. ## 1.8 ### The Logical Release diff --git a/gravity-sync.conf.example b/gravity-sync.conf.example index 8388f89..c125d38 100644 --- a/gravity-sync.conf.example +++ b/gravity-sync.conf.example @@ -27,6 +27,7 @@ REMOTE_PASS='' # LOG_PATH='' # SYNCING_LOG='' # CRONJOB_LOG='' +# HISTORY_MD5='' # VERIFY_PASS='' # SKIP_CUSTOM='' diff --git a/gravity-sync.sh b/gravity-sync.sh index c180e9b..689171a 100755 --- a/gravity-sync.sh +++ b/gravity-sync.sh @@ -429,7 +429,7 @@ function smart_gs { then if [ "${PRIDBCHANGE}" != "0" ] then - MESSAGE="Both Sides Gravity Database Changed" + MESSAGE="Both ${GRAVITY_FI} Changed" echo_warn PRIDBDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${GRAVITY_FI}") @@ -437,13 +437,13 @@ function smart_gs { if [ "${PRIDBDATE}" -gt "$SECDBDATE" ] then - MESSAGE="Primary Gravity Database Last Changed" + MESSAGE="Primary ${GRAVITY_FI} Last Changed" echo_info pull_gs_grav PULLRESTART="1" else - MESSAGE="Secondary Gravity Database Last Changed" + MESSAGE="Secondary ${GRAVITY_FI} Last Changed" echo_info push_gs_grav @@ -476,7 +476,7 @@ function smart_gs { then if [ "${PRICLCHANGE}" != "0" ] then - MESSAGE="Both Sides Custom DNS Changed" + MESSAGE="Both ${CUSTOM_DNS} Changed" echo_warn PRICLDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${CUSTOM_DNS}") @@ -484,13 +484,13 @@ function smart_gs { if [ "${PRICLDATE}" -gt "$SECCLDATE" ] then - MESSAGE="Primary Custom DNS Last Changed" + MESSAGE="Primary ${CUSTOM_DNS} Last Changed" echo_info pull_gs_cust PULLRESTART="1" else - MESSAGE="Secondary Custom DNS Last Changed" + MESSAGE="Secondary ${CUSTOM_DNS} Last Changed" echo_info push_gs_cust @@ -668,6 +668,8 @@ function logs_gs { echo_info echo -e "========================================================" + echo -e "Recent Complete ${YELLOW}SMART${NC} Executions" + tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep SMART echo -e "Recent Complete ${YELLOW}PULL${NC} Executions" tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep PULL echo -e "Recent Complete ${YELLOW}PUSH${NC} Executions"