Merge pull request #47 from vmstan/development

2.0.0
This commit is contained in:
Michael Stanclift 2020-07-07 15:38:06 -05:00 committed by GitHub
commit 80205ea801
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 334 additions and 25 deletions

1
.gitignore vendored
View File

@ -2,6 +2,7 @@ gravity-sync.log
gravity-sync.cron
gravity.db.last
gravity-sync.conf
gravity-sync.md5
backup/*.last
backup/*.backup
backup/*.push

View File

@ -14,8 +14,8 @@ Download the latest release from [GitHub](https://github.com/vmstan/gravity-sync
```bash
cd ~
wget https://github.com/vmstan/gravity-sync/archive/v1.8.3.zip
unzip v1.8.3.zip -d gravity-sync
wget https://github.com/vmstan/gravity-sync/archive/v2.0.0.zip
unzip v2.0.0.zip -d gravity-sync
cd gravity-sync
```
@ -97,7 +97,7 @@ Gravity Sync will place logs in the same folder as the script (identified as .cr
Default setting in Gravity Sync is `$HOME/${LOCAL_FOLDR}`
#### `SYNCING_LOG=''`
Gravity Sync will write a timestamp for any completed pull, push or restore job to this file. If you want to change the name of this file, you will also need to adjust the LOG_PATH variable above, otherwise your file will be remove during an `update` operations.
Gravity Sync will write a timestamp for any completed sync, pull, push or restore job to this file. If you want to change the name of this file, you will also need to adjust the LOG_PATH variable above, otherwise your file will be remove during an `update` operations.
Default setting in Gravity Sync is `gravity-sync.log`
@ -108,6 +108,11 @@ This will have an impact to both the `./gravity-sync.sh automate` function and t
Default setting in Gravity Sync is `gravity-sync.cron`
#### `HISTORY_MD5=''`
Gravity Sync will log the file hashes of the previous `smart` task to this file. If you want to change the name of this file, you will also need to adjust the LOG_PATH variable above, otherwise your file will be removed during an `update` operations.
Default setting in Gravity Sync is `gravity-sync.md5`
#### `VERIFY_PASS=''`
Gravity Sync will prompt to verify user interactivity during push, restore, or config operations (that overwrite an existing configuration) with the intention that it prevents someone from accidentally automating in the wrong direction or overwriting data intentionally. If you'd like to automate a push function, or just don't like to be asked twice to do something distructive, then you can opt-out.
@ -173,5 +178,5 @@ If you prefer to still use cron but modify your settings by hand, using the entr
```bash
crontab -e
*/30 * * * * /bin/bash /home/USER/gravity-sync/gravity-sync.sh pull > /home/USER/gravity-sync/gravity-sync.cron
*/30 * * * * /bin/bash /home/USER/gravity-sync/gravity-sync.sh > /home/USER/gravity-sync/gravity-sync.cron
```

View File

@ -1,5 +1,19 @@
# The Changelog
## 2.0
### The Smart Release
**Features**
In this release, Gravity Sync will now detect not only if each component (`gravity.db` and `custom.list`) has changed since the last sync, but also what direction they need to go. It will then initate a `push` and/or `pull` specific to each piece.
Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull of the `gravity.db` then push `custom.list` and finally restart the correct components on each server. It will also now only perform a sync of each component if there are changes within each type to replicate. So if you only make a small change to your Local DNS settings, it doesn't kickoff the larger `gravity.db` replication.
The default command for Gravity Sync is now just `./gravity-sync.sh` -- but you can also run `./gravity-sync.sh smart` if you feel like it, and it'll do the same thing.
This allows you to be more flexible in where you make your configuration changes to block/allow lists and local DNS settings being made on either the primary or secondary, but it's best practice to continue making changes on one side where possible. In the event there are configuration changes to the same element (example, `custom.list` changes at both sides) then Gravity Sync will attempt to determine based on timestamps on what side the last changed happened, in which case the latest changes will be considered authoritative and overwrite the other side. Gravity Sync does not merge the contents of the files when changes happen, it simply overwrites the entire content.
New installs will use the `smart` function by default. Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. This is not required. The previous `./gravity-sync.sh pull` and `./gravity-sync.sh push` commands continue to function as they did previously, with no intention to break this functionality.
## 1.8
### The Logical Release

View File

@ -1 +1 @@
1.8.3
2.0.0

View File

@ -27,6 +27,7 @@ REMOTE_PASS=''
# LOG_PATH=''
# SYNCING_LOG=''
# CRONJOB_LOG=''
# HISTORY_MD5=''
# VERIFY_PASS=''
# SKIP_CUSTOM=''

View File

@ -3,7 +3,7 @@ SCRIPT_START=$SECONDS
# GRAVITY SYNC BY VMSTAN #####################
PROGRAM='Gravity Sync'
VERSION='1.8.3'
VERSION='2.0.0'
# Execute from the home folder of the user who owns it (ex: 'cd ~/gravity-sync')
# For documentation or downloading updates visit https://github.com/vmstan/gravity-sync
@ -24,6 +24,7 @@ BACKUP_FOLD='backup' # must exist as subdirectory in LOCAL_FOLDR
LOG_PATH="$HOME/${LOCAL_FOLDR}" # replace in gravity-sync.conf to overwrite
SYNCING_LOG='gravity-sync.log' # replace in gravity-sync.conf to overwrite
CRONJOB_LOG='gravity-sync.cron' # replace in gravity-sync.conf to overwrite
HISTORY_MD5='gravity-sync.md5' # replace in gravity-sync.conf to overwrite
# Interaction Customization
VERIFY_PASS='0' # replace in gravity-sync.conf to overwrite
@ -135,10 +136,8 @@ function update_gs {
}
# Gravity Core Functions
## Pull Function
function pull_gs {
md5_compare
## Pull Gravity
function pull_gs_grav {
MESSAGE="Backing Up ${GRAVITY_FI} on $HOSTNAME"
echo_stat
cp ${PIHOLE_DIR}/${GRAVITY_FI} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${GRAVITY_FI}.backup >/dev/null 2>&1
@ -158,7 +157,7 @@ function pull_gs {
MESSAGE="Validating Settings of ${GRAVITY_FI}"
echo_stat
GRAVDB_OWN=$(ls -ld ${PIHOLE_DIR}/${GRAVITY_FI} | awk '{print $3 $4}')
if [ "$GRAVDB_OWN" != "piholepihole" ]
then
@ -196,7 +195,10 @@ function pull_gs {
fi
echo_good
}
## Pull Custom
function pull_gs_cust {
if [ "$SKIP_CUSTOM" != '1' ]
then
if [ "$REMOTE_CUSTOM_DNS" == "1" ]
@ -263,7 +265,10 @@ function pull_gs {
echo_good
fi
fi
}
## Pull Reload
function pull_gs_reload {
MESSAGE="Isolating Regeneration Pathways"
echo_info
sleep 1
@ -277,17 +282,22 @@ function pull_gs {
echo_stat
${PIHOLE_BIN} restartdns >/dev/null 2>&1
error_validate
}
## Pull Function
function pull_gs {
md5_compare
pull_gs_grav
pull_gs_cust
pull_gs_reload
logs_export
exit_withchange
}
## Push Function
function push_gs {
md5_compare
intent_validate
## Push Gravity
function push_gs_grav {
MESSAGE="Backing Up ${GRAVITY_FI} from ${REMOTE_HOST}"
echo_stat
RSYNC_REPATH="rsync"
@ -313,7 +323,10 @@ function push_gs {
CMD_TIMEOUT='15'
CMD_REQUESTED="sudo chown pihole:pihole ${PIHOLE_DIR}/${GRAVITY_FI}"
create_sshcmd
}
## Push Custom
function push_gs_cust {
if [ "$SKIP_CUSTOM" != '1' ]
then
if [ "$REMOTE_CUSTOM_DNS" == "1" ]
@ -345,26 +358,171 @@ function push_gs {
create_sshcmd
fi
fi
}
## Push Reload
function push_gs_reload {
MESSAGE="Inverting Tachyon Pulses"
echo_info
sleep 1
MESSAGE="Updating FTLDNS Configuration"
MESSAGE="Updating Remote FTLDNS Configuration"
echo_stat
CMD_TIMEOUT='15'
CMD_REQUESTED="${PIHOLE_BIN} restartdns reloadlists"
create_sshcmd
MESSAGE="Reloading FTLDNS Services"
MESSAGE="Reloading Remote FTLDNS Services"
echo_stat
CMD_TIMEOUT='15'
CMD_REQUESTED="${PIHOLE_BIN} restartdns"
create_sshcmd
}
## Push Function
function push_gs {
md5_compare
intent_validate
push_gs_grav
push_gs_cust
push_gs_reload
logs_export
exit_withchange
}
## Smart Sync Function
function smart_gs {
md5_compare
if [ -f "${LOG_PATH}/${HISTORY_MD5}" ]
then
last_primaryDBMD5=$(sed "1q;d" ${LOG_PATH}/${HISTORY_MD5})
last_secondDBMD5=$(sed "2q;d" ${LOG_PATH}/${HISTORY_MD5})
last_primaryCLMD5=$(sed "3q;d" ${LOG_PATH}/${HISTORY_MD5})
last_secondCLMD5=$(sed "4q;d" ${LOG_PATH}/${HISTORY_MD5})
else
last_primaryDBMD5="0"
last_secondDBMD5="0"
last_primaryCLMD5="0"
last_secondCLMD5="0"
fi
PRIDBCHANGE="0"
SECDBCHANGE="0"
PRICLCHANGE="0"
SECCLCHANGE="0"
if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ]
then
PRIDBCHANGE="1"
fi
if [ "${secondDBMD5}" != "${last_secondDBMD5}" ]
then
SECDBCHANGE="1"
fi
if [ "${PRIDBCHANGE}" == "${SECDBCHANGE}" ]
then
if [ "${PRIDBCHANGE}" != "0" ]
then
MESSAGE="Both ${GRAVITY_FI} Changed"
echo_warn
PRIDBDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${GRAVITY_FI}")
SECDBDATE=$(stat -c %Y ${PIHOLE_DIR}/${GRAVITY_FI})
if [ "${PRIDBDATE}" -gt "$SECDBDATE" ]
then
MESSAGE="Primary ${GRAVITY_FI} Last Changed"
echo_info
pull_gs_grav
PULLRESTART="1"
else
MESSAGE="Secondary ${GRAVITY_FI} Last Changed"
echo_info
push_gs_grav
PUSHRESTART="1"
fi
fi
else
if [ "${PRIDBCHANGE}" != "0" ]
then
pull_gs_grav
PULLRESTART="1"
elif [ "${SECDBCHANGE}" != "0" ]
then
push_gs_grav
PUSHRESTART="1"
fi
fi
if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ]
then
PRICLCHANGE="1"
fi
if [ "${secondCLMD5}" != "${last_secondCLMD5}" ]
then
SECCLCHANGE="1"
fi
if [ "${PRICLCHANGE}" == "${SECCLCHANGE}" ]
then
if [ "${PRICLCHANGE}" != "0" ]
then
MESSAGE="Both ${CUSTOM_DNS} Changed"
echo_warn
PRICLDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${CUSTOM_DNS}")
SECCLDATE=$(stat -c %Y ${PIHOLE_DIR}/${CUSTOM_DNS})
if [ "${PRICLDATE}" -gt "$SECCLDATE" ]
then
MESSAGE="Primary ${CUSTOM_DNS} Last Changed"
echo_info
pull_gs_cust
PULLRESTART="1"
else
MESSAGE="Secondary ${CUSTOM_DNS} Last Changed"
echo_info
push_gs_cust
PUSHRESTART="1"
fi
fi
else
if [ "${PRICLCHANGE}" != "0" ]
then
pull_gs_cust
PULLRESTART="1"
elif [ "${SECCLCHANGE}" != "0" ]
then
push_gs_cust
PUSHRESTART="1"
fi
fi
if [ "$PULLRESTART" == "1" ]
then
pull_gs_reload
fi
if [ "$PUSHRESTART" == "1" ]
then
push_gs_reload
fi
md5_recheck
logs_export
exit_withchange
}
function restore_gs {
@ -487,6 +645,15 @@ function restore_gs {
## Core Logging
### Write Logs Out
function logs_export {
MESSAGE="Saving File Hashes"
echo_stat
rm -f ${LOG_PATH}/${HISTORY_MD5}
echo -e ${primaryDBMD5} >> ${LOG_PATH}/${HISTORY_MD5}
echo -e ${secondDBMD5} >> ${LOG_PATH}/${HISTORY_MD5}
echo -e ${primaryCLMD5} >> ${LOG_PATH}/${HISTORY_MD5}
echo -e ${secondCLMD5} >> ${LOG_PATH}/${HISTORY_MD5}
error_validate
MESSAGE="Logging Successful ${TASKTYPE}"
echo_stat
echo -e $(date) "[${TASKTYPE}]" >> ${LOG_PATH}/${SYNCING_LOG}
@ -501,6 +668,8 @@ function logs_gs {
echo_info
echo -e "========================================================"
echo -e "Recent Complete ${YELLOW}SMART${NC} Executions"
tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep SMART
echo -e "Recent Complete ${YELLOW}PULL${NC} Executions"
tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep PULL
echo -e "Recent Complete ${YELLOW}PUSH${NC} Executions"
@ -935,6 +1104,86 @@ function md5_compare {
fi
}
function md5_recheck {
MESSAGE="Performing Replicator Diagnostics"
echo_info
HASHMARK='0'
MESSAGE="Reanalyzing ${GRAVITY_FI} on ${REMOTE_HOST}"
echo_stat
primaryDBMD5=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "md5sum ${PIHOLE_DIR}/${GRAVITY_FI}" | sed 's/\s.*$//')
error_validate
MESSAGE="Reanalyzing ${GRAVITY_FI} on $HOSTNAME"
echo_stat
secondDBMD5=$(md5sum ${PIHOLE_DIR}/${GRAVITY_FI} | sed 's/\s.*$//')
error_validate
if [ "$primaryDBMD5" == "$secondDBMD5" ]
then
HASHMARK=$((HASHMARK+0))
else
MESSAGE="Differenced ${GRAVITY_FI} Detected"
echo_warn
HASHMARK=$((HASHMARK+1))
fi
if [ "$SKIP_CUSTOM" != '1' ]
then
if [ -f ${PIHOLE_DIR}/${CUSTOM_DNS} ]
then
if ${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} test -e ${PIHOLE_DIR}/${CUSTOM_DNS}
then
REMOTE_CUSTOM_DNS="1"
MESSAGE="Reanalyzing ${CUSTOM_DNS} on ${REMOTE_HOST}"
echo_stat
primaryCLMD5=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "md5sum ${PIHOLE_DIR}/${CUSTOM_DNS} | sed 's/\s.*$//'")
error_validate
MESSAGE="Reanalyzing ${CUSTOM_DNS} on $HOSTNAME"
echo_stat
secondCLMD5=$(md5sum ${PIHOLE_DIR}/${CUSTOM_DNS} | sed 's/\s.*$//')
error_validate
if [ "$primaryCLMD5" == "$secondCLMD5" ]
then
# MESSAGE="${CUSTOM_DNS} Identical"
# echo_info
HASHMARK=$((HASHMARK+0))
else
MESSAGE="Differenced ${CUSTOM_DNS} Detected"
echo_warn
HASHMARK=$((HASHMARK+1))
fi
else
MESSAGE="No ${CUSTOM_DNS} Detected on ${REMOTE_HOST}"
echo_info
fi
else
if ${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} test -e ${PIHOLE_DIR}/${CUSTOM_DNS}
then
REMOTE_CUSTOM_DNS="1"
MESSAGE="${REMOTE_HOST} has ${CUSTOM_DNS}"
HASHMARK=$((HASHMARK+1))
echo_info
fi
MESSAGE="No ${CUSTOM_DNS} Detected on $HOSTNAME"
echo_info
fi
fi
if [ "$HASHMARK" != "0" ]
then
MESSAGE="Replication Checks Failed"
echo_warn
else
MESSAGE="Replication Was Successful"
echo_info
fi
}
## Validate Intent
function intent_validate {
if [ "$VERIFY_PASS" == "0" ]
@ -1111,9 +1360,10 @@ function list_gs_arguments {
echo -e " ${YELLOW}version${NC} Display installed version of ${PROGRAM}"
echo -e ""
echo -e "Replication Options:"
echo -e " ${YELLOW}pull${NC} Sync remote ${GRAVITY_FI} to this server"
echo -e " ${YELLOW}push${NC} Force changes made on this server back"
echo -e " ${YELLOW}restore${NC} Restore ${GRAVITY_FI} on this server"
echo -e " ${YELLOW}sync${NC} Detect changes on each side and bring them together"
echo -e " ${YELLOW}pull${NC} Force remote configuration changes to this server"
echo -e " ${YELLOW}push${NC} Force local configuration made on this server back"
echo -e " ${YELLOW}restore${NC} Restore the ${GRAVITY_FI} on this server"
echo -e " ${YELLOW}compare${NC} Just check for differences"
echo -e ""
echo -e "Debug Options:"
@ -1207,7 +1457,7 @@ function task_automate {
MESSAGE="Saving New Automation"
echo_stat
(crontab -l 2>/dev/null; echo "*/${INPUT_AUTO_FREQ} * * * * ${BASH_PATH} $HOME/${LOCAL_FOLDR}/${GS_FILENAME} pull > ${LOG_PATH}/${CRONJOB_LOG}") | crontab -
(crontab -l 2>/dev/null; echo "*/${INPUT_AUTO_FREQ} * * * * ${BASH_PATH} $HOME/${LOCAL_FOLDR}/${GS_FILENAME} > ${LOG_PATH}/${CRONJOB_LOG}") | crontab -
error_validate
fi
exit_withchange
@ -1219,7 +1469,7 @@ function clear_cron {
echo_stat
crontab -l > cronjob-old.tmp
sed '/.sh pull/d' cronjob-old.tmp > cronjob-new.tmp
sed '/${GS_FILENAME}/d' cronjob-old.tmp > cronjob-new.tmp
crontab cronjob-new.tmp 2>/dev/null
error_validate
rm cronjob-old.tmp
@ -1383,11 +1633,49 @@ function root_check {
case $# in
0)
task_invalid
TASKTYPE='SMART'
MESSAGE="${MESSAGE}: ${TASKTYPE} Requested"
echo_good
import_gs
validate_gs_folders
validate_ph_folders
validate_os_sshpass
smart_gs
exit
;;
1)
case $1 in
sync)
TASKTYPE='SMART'
MESSAGE="${MESSAGE}: ${TASKTYPE} Requested"
echo_good
import_gs
validate_gs_folders
validate_ph_folders
validate_os_sshpass
smart_gs
exit
;;
smart)
TASKTYPE='SMART'
MESSAGE="${MESSAGE}: ${TASKTYPE} Requested"
echo_good
import_gs
validate_gs_folders
validate_ph_folders
validate_os_sshpass
smart_gs
exit
;;
pull)
TASKTYPE='PULL'
MESSAGE="${MESSAGE}: ${TASKTYPE} Requested"