Merge pull request #47 from vmstan/development

2.0.0

.gitignore

@@ -2,6 +2,7 @@ gravity-sync.log
 gravity-sync.cron
 gravity.db.last
 gravity-sync.conf
+gravity-sync.md5
 backup/*.last
 backup/*.backup
 backup/*.push

ADVANCED.md

@@ -14,8 +14,8 @@ Download the latest release from [GitHub](https://github.com/vmstan/gravity-sync
 
 ```bash
 cd ~
-wget https://github.com/vmstan/gravity-sync/archive/v1.8.3.zip
+wget https://github.com/vmstan/gravity-sync/archive/v2.0.0.zip
-unzip v1.8.3.zip -d gravity-sync
+unzip v2.0.0.zip -d gravity-sync
 cd gravity-sync
 ```
 
@@ -97,7 +97,7 @@ Gravity Sync will place logs in the same folder as the script (identified as .cr
 Default setting in Gravity Sync is `$HOME/${LOCAL_FOLDR}`
 
 #### `SYNCING_LOG=''`
-Gravity Sync will write a timestamp for any completed pull, push or restore job to this file. If you want to change the name of this file, you will also need to adjust the LOG_PATH variable above, otherwise your file will be remove during an `update` operations.
+Gravity Sync will write a timestamp for any completed sync, pull, push or restore job to this file. If you want to change the name of this file, you will also need to adjust the LOG_PATH variable above, otherwise your file will be remove during an `update` operations.
 
 Default setting in Gravity Sync is `gravity-sync.log`
 
@@ -108,6 +108,11 @@ This will have an impact to both the `./gravity-sync.sh automate` function and t
 
 Default setting in Gravity Sync is `gravity-sync.cron`
 
+#### `HISTORY_MD5=''`
+Gravity Sync will log the file hashes of the previous `smart` task to this file. If you want to change the name of this file, you will also need to adjust the LOG_PATH variable above, otherwise your file will be removed during an `update` operations.
+
+Default setting in Gravity Sync is `gravity-sync.md5`
+
 #### `VERIFY_PASS=''`
 Gravity Sync will prompt to verify user interactivity during push, restore, or config operations (that overwrite an existing configuration) with the intention that it prevents someone from accidentally automating in the wrong direction or overwriting data intentionally. If you'd like to automate a push function, or just don't like to be asked twice to do something distructive, then you can opt-out.
 
@@ -173,5 +178,5 @@ If you prefer to still use cron but modify your settings by hand, using the entr
 
 ```bash
 crontab -e
-*/30 * * * * /bin/bash /home/USER/gravity-sync/gravity-sync.sh pull > /home/USER/gravity-sync/gravity-sync.cron
+*/30 * * * * /bin/bash /home/USER/gravity-sync/gravity-sync.sh > /home/USER/gravity-sync/gravity-sync.cron
 ```

CHANGELOG.md

@@ -1,5 +1,19 @@
 # The Changelog
 
+## 2.0
+### The Smart Release
+
+**Features**
+In this release, Gravity Sync will now detect not only if each component (`gravity.db` and `custom.list`) has changed since the last sync, but also what direction they need to go. It will then initate a `push` and/or `pull` specific to each piece.
+
+Example: If the `gravity.db` has been modified on the primary Pi-hole, but the `custom.list` file has been changed on the secondary, Gravity Sync will now do a pull of the `gravity.db` then push `custom.list` and finally restart the correct components on each server. It will also now only perform a sync of each component if there are changes within each type to replicate. So if you only make a small change to your Local DNS settings, it doesn't kickoff the larger `gravity.db` replication.
+
+The default command for Gravity Sync is now just `./gravity-sync.sh` -- but you can also run `./gravity-sync.sh smart` if you feel like it, and it'll do the same thing.
+
+This allows you to be more flexible in where you make your configuration changes to block/allow lists and local DNS settings being made on either the primary or secondary, but it's best practice to continue making changes on one side where possible. In the event there are configuration changes to the same element (example, `custom.list` changes at both sides) then Gravity Sync will attempt to determine based on timestamps on what side the last changed happened, in which case the latest changes will be considered authoritative and overwrite the other side. Gravity Sync does not merge the contents of the files when changes happen, it simply overwrites the entire content.
+
+New installs will use the `smart` function by default. Existing users who want to use this new method as their standard should run `./gravity-sync.sh automate` function to replace the existing automated `pull` with the new Smart Sync. This is not required. The previous `./gravity-sync.sh pull` and `./gravity-sync.sh push` commands continue to function as they did previously, with no intention to break this functionality.
+
 ## 1.8
 ### The Logical Release
 

VERSION

@@ -1 +1 @@
-1.8.3
+2.0.0

gravity-sync.conf.example

@@ -27,6 +27,7 @@ REMOTE_PASS=''
 # LOG_PATH=''
 # SYNCING_LOG=''
 # CRONJOB_LOG=''
+# HISTORY_MD5=''
 
 # VERIFY_PASS=''
 # SKIP_CUSTOM=''

gravity-sync.sh

@@ -3,7 +3,7 @@ SCRIPT_START=$SECONDS
 
 # GRAVITY SYNC BY VMSTAN #####################
 PROGRAM='Gravity Sync'
-VERSION='1.8.3'
+VERSION='2.0.0'
 
 # Execute from the home folder of the user who owns it (ex: 'cd ~/gravity-sync')
 # For documentation or downloading updates visit https://github.com/vmstan/gravity-sync
@@ -24,6 +24,7 @@ BACKUP_FOLD='backup' 				# must exist as subdirectory in LOCAL_FOLDR
 LOG_PATH="$HOME/${LOCAL_FOLDR}"		# replace in gravity-sync.conf to overwrite
 SYNCING_LOG='gravity-sync.log' 		# replace in gravity-sync.conf to overwrite
 CRONJOB_LOG='gravity-sync.cron' 	# replace in gravity-sync.conf to overwrite
+HISTORY_MD5='gravity-sync.md5'		# replace in gravity-sync.conf to overwrite
 
 # Interaction Customization
 VERIFY_PASS='0'						# replace in gravity-sync.conf to overwrite
@@ -135,10 +136,8 @@ function update_gs {
 }
 
 # Gravity Core Functions
-## Pull Function
+## Pull Gravity
-function pull_gs {
+function pull_gs_grav {
-	md5_compare
-	
 	MESSAGE="Backing Up ${GRAVITY_FI} on $HOSTNAME"
 	echo_stat
 		cp ${PIHOLE_DIR}/${GRAVITY_FI} $HOME/${LOCAL_FOLDR}/${BACKUP_FOLD}/${GRAVITY_FI}.backup >/dev/null 2>&1
@@ -196,7 +195,10 @@ function pull_gs {
 		fi
 
 	echo_good
+}
 
+## Pull Custom
+function pull_gs_cust {
 	if [ "$SKIP_CUSTOM" != '1' ]
 	then	
 		if [ "$REMOTE_CUSTOM_DNS" == "1" ]
@@ -263,7 +265,10 @@ function pull_gs {
 			echo_good
 		fi
 	fi
+}
 
+## Pull Reload
+function pull_gs_reload {
 	MESSAGE="Isolating Regeneration Pathways"
 	echo_info
 		sleep 1	
@@ -277,17 +282,22 @@ function pull_gs {
 	echo_stat
 		${PIHOLE_BIN} restartdns >/dev/null 2>&1
 		error_validate
+}
+
+## Pull Function
+function pull_gs {
+	md5_compare
+	
+	pull_gs_grav
+	pull_gs_cust
+	pull_gs_reload
 	
 	logs_export
 	exit_withchange
 }
 
-## Push Function
+## Push Gravity
-function push_gs {
+function push_gs_grav {
-	md5_compare
-	
-	intent_validate
-
 	MESSAGE="Backing Up ${GRAVITY_FI} from ${REMOTE_HOST}"
 	echo_stat
 		RSYNC_REPATH="rsync"
@@ -313,7 +323,10 @@ function push_gs {
 		CMD_TIMEOUT='15'
 		CMD_REQUESTED="sudo chown pihole:pihole ${PIHOLE_DIR}/${GRAVITY_FI}"
 			create_sshcmd
+}
 
+## Push Custom
+function push_gs_cust {
 	if [ "$SKIP_CUSTOM" != '1' ]
 	then	
 		if [ "$REMOTE_CUSTOM_DNS" == "1" ]
@@ -345,26 +358,171 @@ function push_gs {
 					create_sshcmd	
 		fi	
 	fi
+}
 
+## Push Reload
+function push_gs_reload {
 	MESSAGE="Inverting Tachyon Pulses"
 	echo_info
 		sleep 1	
 
-	MESSAGE="Updating FTLDNS Configuration"
+	MESSAGE="Updating Remote FTLDNS Configuration"
 	echo_stat
 		CMD_TIMEOUT='15'
 		CMD_REQUESTED="${PIHOLE_BIN} restartdns reloadlists"
 			create_sshcmd
 	
-	MESSAGE="Reloading FTLDNS Services"
+	MESSAGE="Reloading Remote FTLDNS Services"
 	echo_stat
 		CMD_TIMEOUT='15'
 		CMD_REQUESTED="${PIHOLE_BIN} restartdns"
 			create_sshcmd
+}
+
+## Push Function
+function push_gs {
+	md5_compare
+	
+	intent_validate
+
+	push_gs_grav
+	push_gs_cust
+	push_gs_reload
 
 	logs_export
 	exit_withchange
+}
 
+## Smart Sync Function
+function smart_gs {
+	md5_compare
+
+	if [ -f "${LOG_PATH}/${HISTORY_MD5}" ]
+	then
+		last_primaryDBMD5=$(sed "1q;d" ${LOG_PATH}/${HISTORY_MD5})
+		last_secondDBMD5=$(sed "2q;d" ${LOG_PATH}/${HISTORY_MD5})
+		last_primaryCLMD5=$(sed "3q;d" ${LOG_PATH}/${HISTORY_MD5})
+		last_secondCLMD5=$(sed "4q;d" ${LOG_PATH}/${HISTORY_MD5})
+	else
+		last_primaryDBMD5="0"
+		last_secondDBMD5="0"
+		last_primaryCLMD5="0"
+		last_secondCLMD5="0"
+	fi
+
+	PRIDBCHANGE="0"
+	SECDBCHANGE="0"
+	PRICLCHANGE="0"
+	SECCLCHANGE="0"
+	
+	if [ "${primaryDBMD5}" != "${last_primaryDBMD5}" ]
+	then
+		PRIDBCHANGE="1"
+	fi
+	
+	if [ "${secondDBMD5}" != "${last_secondDBMD5}" ]
+	then
+		SECDBCHANGE="1"
+	fi
+
+	if [ "${PRIDBCHANGE}" == "${SECDBCHANGE}" ]
+	then
+		if [ "${PRIDBCHANGE}" != "0" ]
+		then
+			MESSAGE="Both ${GRAVITY_FI} Changed"
+			echo_warn
+
+			PRIDBDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${GRAVITY_FI}")
+			SECDBDATE=$(stat -c %Y ${PIHOLE_DIR}/${GRAVITY_FI})
+
+				if [ "${PRIDBDATE}" -gt "$SECDBDATE" ]
+				then
+					MESSAGE="Primary ${GRAVITY_FI} Last Changed"
+					echo_info
+
+					pull_gs_grav
+					PULLRESTART="1"
+				else
+					MESSAGE="Secondary ${GRAVITY_FI} Last Changed"
+					echo_info
+
+					push_gs_grav
+					PUSHRESTART="1"
+				fi
+		fi
+	else
+		if [ "${PRIDBCHANGE}" != "0" ]
+		then
+			pull_gs_grav
+			PULLRESTART="1"
+		elif [ "${SECDBCHANGE}" != "0" ]
+		then
+			push_gs_grav
+			PUSHRESTART="1"
+		fi
+	fi
+
+	if [ "${primaryCLMD5}" != "${last_primaryCLMD5}" ]
+	then
+		PRICLCHANGE="1"
+	fi
+	
+	if [ "${secondCLMD5}" != "${last_secondCLMD5}" ]
+	then
+		SECCLCHANGE="1"
+	fi
+
+	if [ "${PRICLCHANGE}" == "${SECCLCHANGE}" ]
+	then
+		if [ "${PRICLCHANGE}" != "0" ]
+		then
+			MESSAGE="Both ${CUSTOM_DNS} Changed"
+			echo_warn
+
+			PRICLDATE=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "stat -c %Y ${PIHOLE_DIR}/${CUSTOM_DNS}")
+			SECCLDATE=$(stat -c %Y ${PIHOLE_DIR}/${CUSTOM_DNS})
+
+				if [ "${PRICLDATE}" -gt "$SECCLDATE" ]
+				then
+					MESSAGE="Primary ${CUSTOM_DNS} Last Changed"
+					echo_info
+
+					pull_gs_cust
+					PULLRESTART="1"
+				else
+					MESSAGE="Secondary ${CUSTOM_DNS} Last Changed"
+					echo_info
+
+					push_gs_cust
+					PUSHRESTART="1"
+				fi
+		fi
+	else
+		if [ "${PRICLCHANGE}" != "0" ]
+		then
+			pull_gs_cust
+			PULLRESTART="1"
+		elif [ "${SECCLCHANGE}" != "0" ]
+		then
+			push_gs_cust
+			PUSHRESTART="1"
+		fi
+	fi
+
+	if [ "$PULLRESTART" == "1" ]
+	then
+		pull_gs_reload
+	fi
+
+	if [ "$PUSHRESTART" == "1" ]
+	then
+		push_gs_reload
+	fi
+
+	md5_recheck
+
+	logs_export
+	exit_withchange
 }
 
 function restore_gs {
@@ -487,6 +645,15 @@ function restore_gs {
 ## Core Logging
 ### Write Logs Out
 function logs_export {
+	MESSAGE="Saving File Hashes"
+	echo_stat
+		rm -f ${LOG_PATH}/${HISTORY_MD5}
+		echo -e ${primaryDBMD5} >> ${LOG_PATH}/${HISTORY_MD5}
+		echo -e ${secondDBMD5} >> ${LOG_PATH}/${HISTORY_MD5}
+		echo -e ${primaryCLMD5} >> ${LOG_PATH}/${HISTORY_MD5}
+		echo -e ${secondCLMD5} >> ${LOG_PATH}/${HISTORY_MD5}
+			error_validate
+
 	MESSAGE="Logging Successful ${TASKTYPE}"
 	echo_stat
 		echo -e $(date) "[${TASKTYPE}]" >> ${LOG_PATH}/${SYNCING_LOG}
@@ -501,6 +668,8 @@ function logs_gs {
 	echo_info
 
 	echo -e "========================================================"
+	echo -e "Recent Complete ${YELLOW}SMART${NC} Executions"
+		tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep SMART
 	echo -e "Recent Complete ${YELLOW}PULL${NC} Executions"
 		tail -n 7 "${LOG_PATH}/${SYNCING_LOG}" | grep PULL
 	echo -e "Recent Complete ${YELLOW}PUSH${NC} Executions"
@@ -935,6 +1104,86 @@ function md5_compare {
 	fi
 }
 
+function md5_recheck {
+	MESSAGE="Performing Replicator Diagnostics"
+	echo_info
+	
+	HASHMARK='0'
+
+	MESSAGE="Reanalyzing ${GRAVITY_FI} on ${REMOTE_HOST}"
+	echo_stat
+	primaryDBMD5=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "md5sum ${PIHOLE_DIR}/${GRAVITY_FI}" | sed 's/\s.*$//') 
+		error_validate
+	
+	MESSAGE="Reanalyzing ${GRAVITY_FI} on $HOSTNAME"
+	echo_stat
+	secondDBMD5=$(md5sum ${PIHOLE_DIR}/${GRAVITY_FI} | sed 's/\s.*$//')
+		error_validate
+	
+	if [ "$primaryDBMD5" == "$secondDBMD5" ]
+	then
+		HASHMARK=$((HASHMARK+0))
+	else
+		MESSAGE="Differenced ${GRAVITY_FI} Detected"
+		echo_warn
+		HASHMARK=$((HASHMARK+1))
+	fi
+
+	if [ "$SKIP_CUSTOM" != '1' ]
+	then
+		if [ -f ${PIHOLE_DIR}/${CUSTOM_DNS} ]
+		then
+			if ${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} test -e ${PIHOLE_DIR}/${CUSTOM_DNS}
+			then
+				REMOTE_CUSTOM_DNS="1"
+				MESSAGE="Reanalyzing ${CUSTOM_DNS} on ${REMOTE_HOST}"
+				echo_stat
+
+				primaryCLMD5=$(${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} "md5sum ${PIHOLE_DIR}/${CUSTOM_DNS} | sed 's/\s.*$//'") 
+					error_validate
+				
+				MESSAGE="Reanalyzing ${CUSTOM_DNS} on $HOSTNAME"
+				echo_stat
+				secondCLMD5=$(md5sum ${PIHOLE_DIR}/${CUSTOM_DNS} | sed 's/\s.*$//')
+					error_validate
+				
+				if [ "$primaryCLMD5" == "$secondCLMD5" ]
+				then
+					# MESSAGE="${CUSTOM_DNS} Identical"
+					# echo_info
+					HASHMARK=$((HASHMARK+0))
+				else
+					MESSAGE="Differenced ${CUSTOM_DNS} Detected"
+					echo_warn
+					HASHMARK=$((HASHMARK+1))
+				fi
+			else
+				MESSAGE="No ${CUSTOM_DNS} Detected on ${REMOTE_HOST}"
+				echo_info
+			fi
+		else
+			if ${SSHPASSWORD} ${SSH_CMD} -p ${SSH_PORT} -i "$HOME/${SSH_PKIF}" ${REMOTE_USER}@${REMOTE_HOST} test -e ${PIHOLE_DIR}/${CUSTOM_DNS}
+			then
+				REMOTE_CUSTOM_DNS="1"
+				MESSAGE="${REMOTE_HOST} has ${CUSTOM_DNS}"
+				HASHMARK=$((HASHMARK+1))
+				echo_info
+			fi	
+			MESSAGE="No ${CUSTOM_DNS} Detected on $HOSTNAME"
+			echo_info
+		fi
+	fi
+
+	if [ "$HASHMARK" != "0" ]
+	then
+		MESSAGE="Replication Checks Failed"
+		echo_warn
+	else
+		MESSAGE="Replication Was Successful"
+		echo_info
+	fi
+}
+
 ## Validate Intent
 function intent_validate {
 	if [ "$VERIFY_PASS" == "0" ]
@@ -1111,9 +1360,10 @@ function list_gs_arguments {
 	echo -e " ${YELLOW}version${NC}	Display installed version of ${PROGRAM}"
 	echo -e ""
 	echo -e "Replication Options:"
-	echo -e " ${YELLOW}pull${NC}		Sync remote ${GRAVITY_FI} to this server"
+	echo -e " ${YELLOW}sync${NC}		Detect changes on each side and bring them together"
-	echo -e " ${YELLOW}push${NC}		Force changes made on this server back"
+	echo -e " ${YELLOW}pull${NC}		Force remote configuration changes to this server"
-	echo -e " ${YELLOW}restore${NC}	Restore ${GRAVITY_FI} on this server"
+	echo -e " ${YELLOW}push${NC}		Force local configuration made on this server back"
+	echo -e " ${YELLOW}restore${NC}	Restore the ${GRAVITY_FI} on this server"
 	echo -e " ${YELLOW}compare${NC}	Just check for differences"
 	echo -e ""
 	echo -e "Debug Options:"
@@ -1207,7 +1457,7 @@ function task_automate {
 
 		MESSAGE="Saving New Automation"
 		echo_stat
-		(crontab -l 2>/dev/null; echo "*/${INPUT_AUTO_FREQ} * * * * ${BASH_PATH} $HOME/${LOCAL_FOLDR}/${GS_FILENAME} pull > ${LOG_PATH}/${CRONJOB_LOG}") | crontab -
+		(crontab -l 2>/dev/null; echo "*/${INPUT_AUTO_FREQ} * * * * ${BASH_PATH} $HOME/${LOCAL_FOLDR}/${GS_FILENAME} > ${LOG_PATH}/${CRONJOB_LOG}") | crontab -
 			error_validate
 	fi
 	exit_withchange
@@ -1219,7 +1469,7 @@ function clear_cron {
 	echo_stat
 
 	crontab -l > cronjob-old.tmp
-	sed '/.sh pull/d' cronjob-old.tmp > cronjob-new.tmp
+	sed '/${GS_FILENAME}/d' cronjob-old.tmp > cronjob-new.tmp
 	crontab cronjob-new.tmp 2>/dev/null
 		error_validate
 	rm cronjob-old.tmp
@@ -1383,11 +1633,49 @@ function root_check {
 case $# in
 	
 	0)
-		task_invalid
+		TASKTYPE='SMART'
+		MESSAGE="${MESSAGE}: ${TASKTYPE} Requested"
+		echo_good
+
+		import_gs
+		validate_gs_folders
+		validate_ph_folders
+		validate_os_sshpass
+
+		smart_gs
+		exit
 	;;
 	
 	1)
    		case $1 in
+		   sync)
+				TASKTYPE='SMART'
+				MESSAGE="${MESSAGE}: ${TASKTYPE} Requested"
+				echo_good
+
+				import_gs
+				validate_gs_folders
+				validate_ph_folders
+				validate_os_sshpass
+
+				smart_gs
+				exit
+ 			;;
+
+			smart)
+				TASKTYPE='SMART'
+				MESSAGE="${MESSAGE}: ${TASKTYPE} Requested"
+				echo_good
+
+				import_gs
+				validate_gs_folders
+				validate_ph_folders
+				validate_os_sshpass
+
+				smart_gs
+				exit
+ 			;;
+
    	 		pull)
 				TASKTYPE='PULL'
 				MESSAGE="${MESSAGE}: ${TASKTYPE} Requested"