From 376d029fc82d1e79ff55736d8e252081f3df4e7a Mon Sep 17 00:00:00 2001
From: David Schroeder <david@schroedercity.com>
Date: Sat, 25 Jul 2026 11:46:04 -0500
Subject: [PATCH] update

---
 README.md           |   8 +-
 defaults.inc        |   2 +-
 proxmenu-scripts.sh | 330 ++++++++++++++++++++++++++++++++++++++++----
 run.sh              | 138 +++++++++---------
 4 files changed, 380 insertions(+), 98 deletions(-)

diff --git a/README.md b/README.md
index 5c7b259..48a9708 100644
--- a/README.md
+++ b/README.md
@@ -11,8 +11,10 @@ Run as `root` on a Proxmox VE host:
 bash <(curl -fsSL https://go.scity.us/install-tapm)
 ```
 
-The installed launcher loads the shared iDSSYS defaults, checks for updates,
-and opens the interactive menu.
+The installed launcher loads the shared iDSSYS defaults and opens the
+interactive menu. Update availability is checked in the background and cached;
+updates are installed only when explicitly selected or requested with
+`tapm update`.
 
 ## Direct actions
 
@@ -29,7 +31,7 @@ proxmenux     Install or open ProxMenux
 virtio       Download current VirtIO drivers
 sentinelone  Install the SentinelOne agent
 screenconnect Install the ScreenConnect agent
-restart      Restart local Proxmox services
+restart      Restart core local Proxmox management services
 cpu          Detect and optionally apply a migration-safe CPU model
 maintenance  Toggle local HA maintenance mode
 keepalived   Deploy Keepalived across the cluster
diff --git a/defaults.inc b/defaults.inc
index 037aaa2..0bd7e01 100755
--- a/defaults.inc
+++ b/defaults.inc
@@ -3,7 +3,7 @@
 
 action="${1:-}"
 FOLDER='/opt/idssys/ta-proxmenu'
-VERS='2026.7.25-15'
+VERS='2026.7.25-16'
 
 noupdate=' '
 
diff --git a/proxmenu-scripts.sh b/proxmenu-scripts.sh
index 7ca5c3a..01e4b0c 100755
--- a/proxmenu-scripts.sh
+++ b/proxmenu-scripts.sh
@@ -235,24 +235,88 @@ DETECT_CPU(){
     
 }
 
-RESTART_PVE_SERVICES(){
-  if [ "${1}" == "" ]; then
-	  echo -en "${idsCL[LightCyan]}Would you like to restart all Proxmox services on the local host (Y/n)?${idsCL[Default]} "
-	  read -n 1 choice
-  else
-    choice=${1}
-  fi
-	case "${choice}" in
-		[Nn]) echo;;
-		*) echo
-			echo -en "\n${idsCL[Yellow]}Restarting services ... "
-			#systemctl restart pve-cluster pvedaemon pvestatd pveproxy pve-ha-lrm pve-ha-crm
-			systemctl restart pve-cluster pvedaemon pvestatd pveproxy
-			echo -e "${idsCL[Green]}Done${idsCL[Default]}"
-			echo -e "\n${idsCL[Green]}This hosts Proxmox services have been restarted${idsCL[Default]}\n"
-			FINISH_ACTION
-		;;
-	esac
+RESTART_SERVICE_GROUP() {
+	local description="$1"
+	shift
+	local -a services=("$@")
+	local service
+	local failed=0
+	local choice
+
+	if (( ${RESTART_ASSUME_YES:-0} == 1 )); then
+		choice="y"
+	else
+		echo -en "${idsCL[LightCyan]}Restart ${description} on this host (Y/n)?${idsCL[Default]} "
+		read -n 1 choice
+		echo
+	fi
+	[[ "$choice" =~ ^[Nn]$ ]] && return
+
+	echo -e "\n${idsCL[Yellow]}Restarting ${description}...${idsCL[Default]}"
+	if ! systemctl restart "${services[@]}"; then
+		failed=1
+	fi
+
+	echo
+	for service in "${services[@]}"; do
+		if systemctl is-active --quiet "$service"; then
+			echo -e " ${idsCL[Green]}[active]${idsCL[Default]} ${service}"
+		else
+			echo -e " ${idsCL[Red]}[failed]${idsCL[Default]} ${service}"
+			failed=1
+		fi
+	done
+
+	if (( failed == 0 )); then
+		echo -e "\n${idsCL[Green]}Service restart completed successfully.${idsCL[Default]}"
+	else
+		echo -e "\n${idsCL[Red]}One or more services failed to restart.${idsCL[Default]}"
+		echo "Review: journalctl -u <service> --since '-10 minutes'"
+	fi
+
+	FINISH_ACTION
+}
+
+RESTART_CLUSTER_FILESYSTEM() {
+	local choice
+
+	echo -e "${idsCL[LightYellow]}This temporarily interrupts /etc/pve (pmxcfs) and cluster configuration access.${idsCL[Default]}"
+	echo -e "${idsCL[LightYellow]}It does not restart Corosync or the HA daemons.${idsCL[Default]}"
+
+	if systemctl is-active --quiet corosync &&
+		! timeout 5 pvecm status 2>/dev/null |
+		grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$'; then
+		echo -e "\n${idsCL[Red]}The cluster is not quorate. pve-cluster will not be restarted.${idsCL[Default]}"
+		ENTER2CONTINUE
+		return
+	fi
+
+	echo -en "\n${idsCL[LightCyan]}Restart pve-cluster on this host (y/N)?${idsCL[Default]} "
+	read -n 1 choice
+	echo
+	[[ "$choice" =~ ^[Yy]$ ]] || return
+
+	if systemctl restart pve-cluster &&
+		systemctl is-active --quiet pve-cluster &&
+		timeout 15 bash -c 'until test -d /etc/pve/nodes; do sleep 1; done'; then
+		echo -e "\n${idsCL[Green]}pve-cluster restarted and /etc/pve is available.${idsCL[Default]}"
+	else
+		echo -e "\n${idsCL[Red]}pve-cluster did not recover normally.${idsCL[Default]}"
+		echo "Review: journalctl -u pve-cluster --since '-10 minutes'"
+	fi
+
+	ENTER2CONTINUE
+}
+
+RESTART_PVE_SERVICES() {
+	local requested_choice="${1:-}"
+	local RESTART_ASSUME_YES=0
+
+	[[ "$requested_choice" =~ ^[Nn]$ ]] && return
+	[[ "$requested_choice" =~ ^[Yy]$ ]] && RESTART_ASSUME_YES=1
+
+	RESTART_SERVICE_GROUP "core Proxmox management services" \
+		pvedaemon pveproxy pvestatd pvescheduler
 }
 
 MAINTENANCE_MODE(){
@@ -295,12 +359,151 @@ INSTALL_KEEPALIVE() {
 	esac
 }
 
+UPDATE_CACHE_DIR='/var/cache/ta-proxmenu'
+UPDATE_CACHE_FILE="${UPDATE_CACHE_DIR}/update-status"
+UPDATE_CACHE_SECONDS=14400
+UPDATE_CHECK_PID=''
+UPDATE_STATUS='unknown'
+UPDATE_REMOTE_COMMIT=''
+
+WRITE_UPDATE_CACHE() {
+	local status="$1"
+	local branch="$2"
+	local local_commit="$3"
+	local remote_commit="$4"
+	local cache_temp
+
+	mkdir -p "$UPDATE_CACHE_DIR" || return 1
+	cache_temp="$(mktemp "${UPDATE_CACHE_DIR}/.update-status.XXXXXX")" || return 1
+	printf '%s|%s|%s|%s|%s\n' \
+		"$(date +%s)" "$branch" "$local_commit" "$remote_commit" "$status" \
+		>"$cache_temp"
+	chmod 0644 "$cache_temp"
+	mv -f "$cache_temp" "$UPDATE_CACHE_FILE"
+}
+
+UPDATE_CHECK_WORKER() {
+	local branch
+	local local_commit
+	local remote_commit
+	local status
+
+	branch="$(git -C "$FOLDER" branch --show-current 2>/dev/null)"
+	local_commit="$(git -C "$FOLDER" rev-parse HEAD 2>/dev/null)"
+
+	if [[ -z "$branch" || -z "$local_commit" ]]; then
+		WRITE_UPDATE_CACHE "unavailable" "$branch" "$local_commit" ""
+		return
+	fi
+
+	remote_commit="$(git -C "$FOLDER" ls-remote origin "refs/heads/${branch}" 2>/dev/null | cut -f1)"
+	if [[ -z "$remote_commit" ]]; then
+		status="unavailable"
+	elif [[ "$local_commit" == "$remote_commit" ]]; then
+		status="current"
+	else
+		status="available"
+	fi
+
+	WRITE_UPDATE_CACHE "$status" "$branch" "$local_commit" "$remote_commit"
+}
+
+START_UPDATE_CHECK() {
+	if [[ -n "$UPDATE_CHECK_PID" ]] &&
+		kill -0 "$UPDATE_CHECK_PID" 2>/dev/null; then
+		return
+	fi
+	UPDATE_CHECK_WORKER >/dev/null 2>&1 &
+	UPDATE_CHECK_PID="$!"
+}
+
+LOAD_UPDATE_STATUS() {
+	local checked_at
+	local cached_branch
+	local cached_local
+	local cached_remote
+	local cached_status
+	local current_branch
+	local current_commit
+	local now
+
+	UPDATE_STATUS='unknown'
+	UPDATE_REMOTE_COMMIT=''
+	current_branch="$(git -C "$FOLDER" branch --show-current 2>/dev/null)"
+	current_commit="$(git -C "$FOLDER" rev-parse HEAD 2>/dev/null)"
+	now="$(date +%s)"
+
+	if [[ -r "$UPDATE_CACHE_FILE" ]]; then
+		IFS='|' read -r checked_at cached_branch cached_local cached_remote cached_status \
+			<"$UPDATE_CACHE_FILE"
+		if [[ "$checked_at" =~ ^[0-9]+$ ]] &&
+			(( now - checked_at < UPDATE_CACHE_SECONDS )) &&
+			[[ "$cached_branch" == "$current_branch" ]] &&
+			[[ "$cached_local" == "$current_commit" ]] &&
+			[[ "$cached_status" =~ ^(current|available|unavailable)$ ]]; then
+			UPDATE_STATUS="$cached_status"
+			UPDATE_REMOTE_COMMIT="$cached_remote"
+			return
+		fi
+	fi
+
+	UPDATE_STATUS='checking'
+	START_UPDATE_CHECK
+}
+
+FORCE_UPDATE_CHECK() {
+	local attempts=0
+
+	rm -f "$UPDATE_CACHE_FILE"
+	UPDATE_CHECK_PID=''
+	START_UPDATE_CHECK
+
+	echo -en "${idsCL[LightCyan]}Checking current branch for updates"
+	while (( attempts < 20 )); do
+		sleep 0.5
+		LOAD_UPDATE_STATUS
+		[[ "$UPDATE_STATUS" != "checking" ]] && break
+		echo -n "."
+		((attempts++))
+	done
+	echo -e "${idsCL[Default]}"
+
+	case "$UPDATE_STATUS" in
+		available)
+			echo -e "${idsCL[LightYellow]}An update is available for the current branch.${idsCL[Default]}"
+			;;
+		current)
+			echo -e "${idsCL[Green]}TA-ProxMenu is current.${idsCL[Default]}"
+			;;
+		*)
+			echo -e "${idsCL[Red]}The update status could not be determined.${idsCL[Default]}"
+			;;
+	esac
+
+	ENTER2CONTINUE
+}
+
 
 
 MENU_HEADER() {
+	local version_display
+
+	LOAD_UPDATE_STATUS
+	case "$UPDATE_STATUS" in
+		available)
+			version_display="${idsCL[LightYellow]}${VERS}  ** UPDATE AVAILABLE **${idsCL[Default]}"
+			;;
+		checking)
+			version_display="${idsCL[LightCyan]}${VERS}  (checking for updates)${idsCL[Default]}"
+			;;
+		*)
+			version_display="${idsCL[White]}${VERS}${idsCL[Default]}"
+			;;
+	esac
+
 	clear
 	echo
-	echo -e " ${idsCL[Green]}TA-ProxMenu - Proxmox Setup Scripts${idsCL[Default]}                          ${VERS}"
+	echo -e " ${idsCL[Green]}TA-ProxMenu - Proxmox Setup Scripts${idsCL[Default]}      ${version_display}"
 	echo -e "${idsCL[Green]}---------------------------------------------------------------------------${idsCL[Default]}"
 	echo -e " Hostname:    ${idsCL[Cyan]}$(hostname -s)${idsCL[Default]}"
 	echo -e " IP Address:  ${idsCL[Cyan]}${RNIP:-Unavailable}${idsCL[Default]}"
@@ -340,7 +543,12 @@ SELECT_MENU() {
 			echo " ↑/↓ Navigate   Enter Select   Number Quick Select   Q Quit"
 		fi
 
-		IFS= read -rsn1 key
+		key=""
+		if [[ "$UPDATE_STATUS" == "checking" ]]; then
+			IFS= read -rsn1 -t 1 key || continue
+		else
+			IFS= read -rsn1 key
+		fi
 		case "$key" in
 			"")
 				MENU_SELECTION="${values_ref[$selected]}"
@@ -494,7 +702,7 @@ MONITORING_MENU() {
 
 CLUSTER_MENU() {
 	local -a labels
-	local -a values=("maintenance" "restart" "keepalived")
+	local -a values=("maintenance" "services" "keepalived")
 
 	while true; do
 		if ha-manager status | grep -F "$(hostname -s)" | grep -q "maintenance mode"; then
@@ -503,7 +711,7 @@ CLUSTER_MENU() {
 			labels=("Put this host into maintenance mode and evacuate guests")
 		fi
 
-		labels+=("Restart local Proxmox services")
+		labels+=("Proxmox service recovery")
 		dpkg-query -W -f='${Status}' keepalived 2>/dev/null | grep -q "install ok installed" &&
 			labels+=("Deploy/reconfigure Keepalived (installed locally)") ||
 			labels+=("Deploy Keepalived on all cluster hosts")
@@ -511,7 +719,7 @@ CLUSTER_MENU() {
 		SELECT_MENU "Cluster & Maintenance" labels values
 		case "$MENU_SELECTION" in
 			maintenance) MAINTENANCE_MODE;;
-			restart) RESTART_PVE_SERVICES;;
+			services) SERVICE_RECOVERY_MENU;;
 			keepalived) INSTALL_KEEPALIVE;;
 			back) return;;
 			quit) EXIT1; exit 0;;
@@ -519,17 +727,81 @@ CLUSTER_MENU() {
 	done
 }
 
-UTILITIES_MENU() {
-	local -a labels=("Check for script updates" "Version and installation information")
-	local -a values=("update" "about")
+SERVICE_RECOVERY_MENU() {
+	local -a labels=(
+		"Restart Web UI and API (pveproxy, pvedaemon)"
+		"Restart statistics collection (pvestatd)"
+		"Restart task scheduler (pvescheduler)"
+		"Restart core management services"
+		"Restart cluster filesystem (pve-cluster)"
+	)
+	local -a values=("web" "statistics" "scheduler" "core" "clusterfs")
 
 	while true; do
+		SELECT_MENU "Proxmox Service Recovery" labels values
+		case "$MENU_SELECTION" in
+			web)
+				RESTART_SERVICE_GROUP "Web UI and API services" pveproxy pvedaemon
+				;;
+			statistics)
+				RESTART_SERVICE_GROUP "statistics collection" pvestatd
+				;;
+			scheduler)
+				RESTART_SERVICE_GROUP "task scheduler" pvescheduler
+				;;
+			core)
+				RESTART_PVE_SERVICES
+				;;
+			clusterfs)
+				RESTART_CLUSTER_FILESYSTEM
+				;;
+			back) return;;
+			quit) EXIT1; exit 0;;
+		esac
+	done
+}
+
+UTILITIES_MENU() {
+	local -a labels
+	local -a values=("install_update" "check_update" "about")
+	local choice
+
+	while true; do
+		LOAD_UPDATE_STATUS
+		case "$UPDATE_STATUS" in
+			available)
+				labels=("Install available update")
+				;;
+			current)
+				labels=("TA-ProxMenu is current")
+				;;
+			checking)
+				labels=("Update check in progress")
+				;;
+			*)
+				labels=("Update status unavailable")
+				;;
+		esac
+		labels+=("Check again now" "Version and installation information")
+
 		SELECT_MENU "Utilities" labels values
 		case "$MENU_SELECTION" in
-			update)
-				/opt/idssys/ta-proxmenu/run.sh update
-				read -r -p " Press ENTER to return..." _
+			install_update)
+				if [[ "$UPDATE_STATUS" != "available" ]]; then
+					echo -e "\n${idsCL[LightCyan]}No available update is currently detected.${idsCL[Default]}"
+					ENTER2CONTINUE
+					continue
+				fi
+				echo -en "\n${idsCL[LightCyan]}Install the update for the current branch (y/N)?${idsCL[Default]} "
+				read -n 1 choice
+				echo
+				if [[ "$choice" =~ ^[Yy]$ ]] &&
+					/opt/idssys/ta-proxmenu/run.sh update; then
+					exec /opt/idssys/ta-proxmenu/run.sh
+				fi
+				ENTER2CONTINUE
 				;;
+			check_update) FORCE_UPDATE_CHECK;;
 			about) SHOW_ABOUT;;
 			back) return;;
 			quit) EXIT1; exit 0;;
diff --git a/run.sh b/run.sh
index bce38d6..7b93878 100755
--- a/run.sh
+++ b/run.sh
@@ -1,79 +1,87 @@
 #!/usr/bin/env bash
-# TA-Proxmenu preloader
+# TA-ProxMenu preloader
 
 [ "${2:-}" != "q" ] && source /opt/idssys/defaults/colors.inc
 source /opt/idssys/defaults/default.inc
 source /opt/idssys/ta-proxmenu/defaults.inc
 
-if [[ "${noupdate}" != *" ${1:-} "* ]] && [[ "${noupdate}" != *" ${2:-} "* ]]; then
-  if curl -m 3 -s --head  --request GET https://git.scity.us | grep "HTTP/2 200" > /dev/null; then 
-  	if [ "${1}" != "tapm" ]; then
-  		echo -en "${idsCL[LightCyan]}Checking for updates...${idsCL[Default]}"
-  		echo ""
-  		udtd=0
-  	fi
+UPDATE_REPOSITORY() {
+	local repository="$1"
+	local branch="$2"
+	local label="$3"
+	local local_commit
+	local remote_commit
 
-  	if [ "${1}" != "tapm" ]; then
-  		if [ ! -d /opt/idssys/defaults ]; then
-  			git clone https://git.scity.us/voltron/iDS-Defaults.git /opt/idssys/defaults
-  		else
-  			cd /opt/idssys/defaults
-			if [ "$(git rev-parse HEAD)" != "$(git ls-remote origin refs/heads/master | cut -f1)" ]; then
-  				if [ "${1}" != "tapm" ]; then
-  					echo -en "\e[1A";
-  					echo -en "\e[0K\r${idsCL[LightCyan]}Updating iDSSYS-Defaults...${idsCL[Default]}"
-  					udtd=1
-  				fi
-  				git fetch origin master >/dev/null 2>&1
-  				git reset --hard origin/master >/dev/null 2>&1
-  				git reflog expire --expire=now --all >/dev/null 2>&1
-  				git repack -ad >/dev/null 2>&1
-  				git prune >/dev/null 2>&1
-  				git pull >/dev/null 2>&1
-  				[ "${1}" != "tapm" ] && echo -e "${idsCL[Green]}Done${idsCL[Default]}"
-  			fi
-  		fi
-  	fi
+	cd "$repository" || return 1
+	local_commit="$(git rev-parse HEAD 2>/dev/null)" || return 1
+	remote_commit="$(git ls-remote origin "refs/heads/${branch}" | cut -f1)"
 
-	cd /opt/idssys/ta-proxmenu
-	current_branch="$(git branch --show-current)"
-	remote_commit=""
-	if [ -n "$current_branch" ]; then
-		remote_commit="$(git ls-remote origin "refs/heads/${current_branch}" | cut -f1)"
+	if [ -z "$remote_commit" ]; then
+		echo -e "${idsCL[Red]}Could not find branch '${branch}' for ${label}${idsCL[Default]}"
+		return 1
 	fi
 
+	if [ "$local_commit" = "$remote_commit" ]; then
+		echo -e "${idsCL[Green]}${label} is current (${branch})${idsCL[Default]}"
+		return 0
+	fi
+
+	echo -en "${idsCL[LightCyan]}Updating ${label} (${branch})...${idsCL[Default]}"
+	if git fetch origin "$branch" >/dev/null 2>&1 &&
+		git reset --hard "origin/${branch}" >/dev/null 2>&1; then
+		echo -e " ${idsCL[Green]}Done${idsCL[Default]}"
+		return 0
+	fi
+
+	echo -e " ${idsCL[Red]}Failed${idsCL[Default]}"
+	return 1
+}
+
+INSTALL_UPDATES() {
+	local current_branch
+	local update_failed=0
+
+	echo -e "${idsCL[LightCyan]}Checking for updates...${idsCL[Default]}"
+	if ! curl --fail --silent --show-error --head \
+		--connect-timeout 3 --max-time 10 https://git.scity.us >/dev/null; then
+		echo -e "${idsCL[Red]}Could not connect to git.scity.us${idsCL[Default]}"
+		return 1
+	fi
+
+	UPDATE_REPOSITORY /opt/idssys/defaults master "iDSSYS Defaults" ||
+		update_failed=1
+
+	current_branch="$(git -C /opt/idssys/ta-proxmenu branch --show-current)"
 	if [ -z "$current_branch" ]; then
-		echo -e "${idsCL[Red]}TA-Proxmenu is in a detached HEAD state; automatic updates were skipped${idsCL[Default]}"
-	elif [ -z "$remote_commit" ]; then
-		echo -e "${idsCL[Red]}Could not find branch '${current_branch}' on the TA-Proxmenu origin; automatic updates were skipped${idsCL[Default]}"
-	elif [ "$(git rev-parse HEAD)" != "$remote_commit" ]; then
-  		if [ "${1}" != "tapm" ]; then
-  			[ ${udtd} -eq 0 ] && echo -en "\e[1A";
-			echo -en "\e[0K\r${idsCL[LightCyan]}Updating TA-Proxmenu (${current_branch})...${idsCL[Default]}"
-  		fi
-		git fetch origin "$current_branch" >/dev/null 2>&1
-		git reset --hard "origin/${current_branch}" >/dev/null 2>&1
-  		git reflog expire --expire=now --all >/dev/null 2>&1
-  		git repack -ad >/dev/null 2>&1
-  		git prune >/dev/null 2>&1
-		
-  		if [ "${1}" != "tapm" ]; then
-  			source /opt/idssys/ta-proxmenu/defaults.inc
-  			# echo -en "\e[1A";
-  			# echo -e "\e[0K\r ${idsCL[Green]}Updated to v${VERS}${idsCL[Default]}"
-  			echo -e " ${idsCL[Green]}Updated to v${VERS}${idsCL[Default]}\n"
-  		fi
-  	elif [ "${1}" != "tapm" ] && [ ${udtd} -eq 0 ]; then
-  		echo -e "\e[1A\e[0K\r ${idsCL[Green]}No updates available${idsCL[Default]}\n"
-  	fi
+		echo -e "${idsCL[Red]}TA-ProxMenu is in a detached HEAD state; update skipped${idsCL[Default]}"
+		update_failed=1
+	else
+		UPDATE_REPOSITORY /opt/idssys/ta-proxmenu "$current_branch" "TA-ProxMenu" ||
+			update_failed=1
+	fi
 
-  else
-  	echo -e "${idsCL[Red]}Could not connect to 'git.scity.us' for updates${idsCL[Default]}"
-  fi
-fi
+	rm -f /var/cache/ta-proxmenu/update-status 2>/dev/null || true
 
-if [ "${1}" != "tapm" ] && [ "${1}" != "update" ] && [ "${1}" != "u" ]; then
-	/opt/idssys/ta-proxmenu/proxmenu-scripts.sh "${1:-}" "${2:-}" "${3:-}" "${4:-}"
-fi
+	if (( update_failed == 0 )); then
+		source /opt/idssys/ta-proxmenu/defaults.inc
+		echo -e "\n${idsCL[Green]}Update check complete. Installed version: ${VERS}${idsCL[Default]}"
+		return 0
+	fi
 
-exit 0
+	return 1
+}
+
+case "${1:-}" in
+	update|u)
+		INSTALL_UPDATES
+		exit $?
+		;;
+	tapm)
+		exit 0
+		;;
+esac
+
+/opt/idssys/ta-proxmenu/proxmenu-scripts.sh \
+	"${1:-}" "${2:-}" "${3:-}" "${4:-}"
+
+exit $?