#!/usr/bin/env bash
# TA-Proxmenu - Proxmox Setup Scripts for TA Use


[ "${2}" != "q" ] && source /opt/idssys/defaults/colors.inc
source /opt/idssys/defaults/default.inc
source /opt/idssys/ta-proxmenu/defaults.inc

INSTALL_PULSE() {
	echo
	bash <(curl -fsSL https://github.com/rcourtman/Pulse/releases/latest/download/install.sh)
	echo
	echo -e "\n${idsCL[Green]}Pulse has been installed${idsCL[Default]}"
	[ ${action-x} ] && exit 0 || ENTER2CONTINUE
}

INSTALL_ACRONIS() {
	read -n 1 -p "Are you sure you wish to install Acronis (Y/n)?" choice
	case "$choice" in
		[Nn]) MAIN_MENU;;
		* )
			echo
			cd /tmp
			wget "https://us5-cloud.acronis.com/bc/api/ams/links/agents/redirect?language=multi&channel=CURRENT&system=linux&architecture=64&productType=enterprise&login=010180ae-63c4-4495-bed0-4ec934c25af9&white_labeled=0" -O ./acronisinstall
			chmod +x ./acronisinstall
			./acronisinstall
			rm -f ./acronisinstall
			echo
			echo -e "\n${idsCL[Green]}Acronis has been installed${idsCL[Default]}"
			[ ${action-x} ] && exit 0 || ENTER2CONTINUE
		;;
	esac
}

INSTALL_PROXMENUX() {
#	read -n 1 -p "Are you sure you wish to install ProxMenux (Y/n)?" choice
#	case "$choice" in 
#	[Nn]) MAIN_MENU;;
#	* )
#		echo
		bash -c "$(wget -qLO - https://raw.githubusercontent.com/MacRimi/ProxMenux/main/install_proxmenux.sh)"
    # systemctl disable --now proxmenux-monitor
		menu
#		echo -e "\n${idsCL[Green]}ProxMenux has been installed${idsCL[Default]}"
#		[ ${action-x} ] && exit 0 || ENTER2CONTINUE
#	esac
}

PROXMENUX_POST_INSTALL() {
	PMFLDR='/usr/local/share/proxmenux/scripts/post_install'
	[ ! -f ${PMFLDR}/customizable_post_install.sh ] && INSTALL_PROXMENUX
	bash ${PMFLDR}/customizable_post_install.sh
	
	touch /opt/.PROXMENUX_POST_INSTALL
	[ -s /etc/apt/sources.list ] && cat /dev/null > /etc/apt/sources.list
	
}

INSTALL_GLANCES() {
	read -n 1 -p "Are you sure you wish to install Glances (Y/n)?" choice
	case "$choice" in 
	[Nn]) echo;;
	* )
		echo
		apt install glances -y
		echo -e "\n${idsCL[Green]}Glances has been installed${idsCL[Default]}"
		[ ${action-x} ] && exit 0 || ENTER2CONTINUE
	esac
}

INSTALL_SCREENCONNECT() {
	read -n 1 -p "Are you sure you wish to install ScreenConnect (Y/n)?" choice
	case "$choice" in 
	[Nn]) echo;;
	* )
		echo
		echo -en "\n${idsCL[LightYellow]}Paste the URL provided from the Build Installer: ${idsCL[Default]}"
		read -e SCURL    
		wget "${SCURL}" -O /tmp/scinstall
		dpkg -i /tmp/scinstall
		apt install --fix-broken -y
		apt remove "connectwis*" -y > /dev/null 2>&1
		dpkg -i /tmp/scinstall
		rm -f /tmp/scinstall
		systemctl disable --now proxmenux-monitor
		echo -e "\n${idsCL[Green]}ScreenConnect has been installed${idsCL[Default]}"
		[ ${action-x} ] && exit 0 || ENTER2CONTINUE
	esac
}

INSTALL_RMM() {
	read -n 1 -p "Are you sure you wish to install RMM (Y/n)?" choice
	case "$choice" in 
	[Nn]) echo;;
	* )
		echo
    
		echo -en "\n${idsCL[LightYellow]}Paste the Linux Server URL provided from the Download Agent screen: ${idsCL[Default]}"
		read -e RMMURL
		wget "${RMMURL}" -O /tmp/rmminstall
		TOKEN="$(echo ${RMMURL} | awk -F 'TKN' '{print $2}' | awk -F '/RUN' '{print $1}')"
		CMD="TOKEN=${TOKEN} bash /tmp/rmminstall"
		eval ${CMD}
		systemctl restart ITSPlatform
		# rm -f /tmp/rmminstall
    
		echo -e "\n${idsCL[Green]}RMM has been installed${idsCL[Default]}"
		[ ${action-x} ] && exit 0 || ENTER2CONTINUE
	esac
}

INSTALL_S1() {
	echo
	echo -en "${idsCL[LightYellow]}Paste the customers SentinelOne Site Token: ${idsCL[Default]}"
	read -e s1token
	cd /tmp
	wget "https://git.scity.us/TAI/files/raw/branch/main/SentinelAgent_linux_x86_64_v26_1_1_31.deb"
	dpkg -i ./SentinelAgent_linux_x86_64*.deb
	/opt/sentinelone/bin/sentinelctl management token set ${s1token}
	/opt/sentinelone/bin/sentinelctl control start
	rm -f ./SentinelAgent_linux_x86_64*.deb
	
	echo -e "\n${idsCL[Green]}SentinelOne Agent has been installed. Make sure its added to a \"DETECT ONLY\" policy${idsCL[Default]}"
	[ ${action-x} ] && exit 0 || ENTER2CONTINUE
}

INSTALL_OMSA() {
	read -n 1 -p "Are you sure you wish to install Dell OpenManage Administrator (Y/n)?" choice
	case "$choice" in 
	[Nn]) echo;;
	* )
		echo
		mkdir /tmp/omsa
		cd /tmp/omsa
		apt install -y gnupg libcurl4t64 libncurses6 libxslt1.1 libgpm2 libtinfo6
		mkdir -p /etc/apt/keyrings
		wget -qO - https://linux.dell.com/repo/pgp_pubkeys/0x1285491434D8786F.asc | gpg --dearmor -o /etc/apt/keyrings/linux.dell.com.gpg
		chmod +r /etc/apt/keyrings/linux.dell.com.gpg
		echo "deb [signed-by=/etc/apt/keyrings/linux.dell.com.gpg] http://linux.dell.com/repo/community/openmanage/11000/jammy jammy main" > /etc/apt/sources.list.d/linux.dell.com.list
		apt update
		wget -c http://archive.ubuntu.com/ubuntu/pool/universe/o/openwsman/libwsman-curl-client-transport1_2.6.5-0ubuntu16_amd64.deb
		wget -c http://archive.ubuntu.com/ubuntu/pool/universe/o/openwsman/libwsman-client4t64_2.6.5-0ubuntu16_amd64.deb
		wget -c http://archive.ubuntu.com/ubuntu/pool/universe/o/openwsman/libwsman1t64_2.6.5-0ubuntu16_amd64.deb
		# wget -c http://http.us.debian.org/debian/pool/main/libx/libxml2/libxml2-16_2.15.1+dfsg-2+b1_amd64.deb
		wget -c http://http.us.debian.org/debian/pool/main/libx/libxml2/libxml2-16_2.15.2+dfsg-0.1_amd64.deb
		wget -c http://archive.ubuntu.com/ubuntu/pool/universe/o/openwsman/libwsman-server1t64_2.6.5-0ubuntu16_amd64.deb
		wget -c http://archive.ubuntu.com/ubuntu/pool/universe/s/sblim-sfcc/libcimcclient0_2.2.8-0ubuntu2_amd64.deb
		wget -c http://archive.ubuntu.com/ubuntu/pool/universe/o/openwsman/openwsman_2.6.5-0ubuntu16_amd64.deb
		wget -c http://archive.ubuntu.com/ubuntu/pool/multiverse/c/cim-schema/cim-schema_2.48.0-0ubuntu1_all.deb
		wget -c http://archive.ubuntu.com/ubuntu/pool/universe/s/sblim-sfc-common/libsfcutil0_1.0.1-0ubuntu4_amd64.deb
		wget -c http://archive.ubuntu.com/ubuntu/pool/multiverse/s/sblim-sfcb/sfcb_1.4.9-0ubuntu7_amd64.deb
		wget -c http://archive.ubuntu.com/ubuntu/pool/universe/s/sblim-cmpi-devel/libcmpicppimpl0_2.0.3-0ubuntu2_amd64.deb
		wget -c http://ftp.us.debian.org/debian/pool/main/o/openssl/libssl1.1_1.1.1w-0+deb11u1_amd64.deb
		dpkg -i libwsman-curl-client-transport1_2.6.5-0ubuntu16_amd64.deb
		dpkg -i libwsman-client4t64_2.6.5-0ubuntu16_amd64.deb
		dpkg -i libxml2-16_2.15.2+dfsg-0.1_amd64.deb
		dpkg -i libwsman1t64_2.6.5-0ubuntu16_amd64.deb
		dpkg -i libwsman-server1t64_2.6.5-0ubuntu16_amd64.deb
		dpkg -i libcimcclient0_2.2.8-0ubuntu2_amd64.deb
		dpkg -i openwsman_2.6.5-0ubuntu16_amd64.deb
		dpkg -i cim-schema_2.48.0-0ubuntu1_all.deb
		dpkg -i libsfcutil0_1.0.1-0ubuntu4_amd64.deb
		dpkg -i sfcb_1.4.9-0ubuntu7_amd64.deb
		dpkg -i libcmpicppimpl0_2.0.3-0ubuntu2_amd64.deb
		dpkg -i libssl1.1_1.1.1w-0+deb11u1_amd64.deb
		apt install -y srvadmin-all
		/opt/dell/srvadmin/sbin/srvadmin-services.sh start
		rm -Rf /tmp/omsa
		
		echo -e "\n${idsCL[Green]}Dell OMSA has been installed${idsCL[Default]}"
		echo -e "\n${idsCL[LightCyan]}Available at: ${idsCL[LightGreen]}https://${RNIP}:1311${idsCL[Default]}"
		[ ${action-x} ] && exit 0 || ENTER2CONTINUE
	esac
}

DOWNLOAD_VIRTIO() {
  
  echo -e "\n${idsCL[LightCyan]}Current \"Stable\" version available for download: ${idsCL[White]}${VIRTIO_FILE}${idsCL[Default]}"
  if [ -f ${dldir}/${VIRTIO_FILE} ]; then
  	echo -en "\n${idsCL[LightRed]}Removing existing download ... "
  	rm -f ${DLDIR}/${VIRTIO_FILE}
  	echo -e "${idsCL[Red]}Done${idsCL[Default]}"
  fi
  wget -q -F -P ${DLDIR} ${VIRTIO_DOWNLOAD_URL} &
  echo -e "\n${idsCL[LightCyan]}Downloading will continue in the background\n"
	[ ${action-x} ] && exit 0 || ENTER2CONTINUE
}

DETECT_CPU(){
  # if [ ! -f /etc/apt/sources.list.d/proxlb.list ]; then
  #   echo "deb https://repo.gyptazy.com/stable /" > /etc/apt/sources.list.d/proxlb.list
  #   wget -O /etc/apt/trusted.gpg.d/proxlb.asc https://repo.gyptazy.com/repository.gpg
  #   apt-get update
  # fi
	if [ ! -f /etc/apt/sources.list.d/gyptazy.list ]; then
		curl https://git.gyptazy.com/api/packages/gyptazy/debian/repository.key -o /etc/apt/keyrings/gyptazy.asc
		echo "deb [signed-by=/etc/apt/keyrings/gyptazy.asc] https://packages.gyptazy.com/api/packages/gyptazy/debian trixie main" | sudo tee -a /etc/apt/sources.list.d/gyptazy.list
		apt update
	fi
	if [ "$(dpkg -l | awk '/proxclmc/ {print }'|wc -l)" -eq 0 ]; then
	  apt -y install proxclmc
  fi

	echo
	proxclmc
	echo
	echo -en "${idsCL[LightCyan]}Would you like to set '${idsCL[LightGreen]}cpu: $(proxclmc --list-only)${idsCL[LightCyan]}' on all VMs (y/N)?${idsCL[Default]} "
	read -n 1 choice
	case "$choice" in 
		[Yy])
			sed -i "/cpu:/c cpu: $(proxclmc --list-only)" /etc/pve/nodes/*/qemu-server/*.conf
			echo
			echo -e "\n${idsCL[Green]}All VM's have been reconfigured\n${idsCL[LightCyan]}This will require the VM's to be powered off and then turned back on in order to take effect${idsCL[Default]}"
			[ ${action-x} ] && exit 0 || ENTER2CONTINUE
		;;
		*) echo;;
	esac
    
}

RESTART_PVE_SERVICES(){
  if [ "${1}" == "" ]; then
	  echo -en "${idsCL[LightCyan]}Would you like to restart all Proxmox services on the local host (Y/n)?${idsCL[Default]} "
	  read -n 1 choice
  else
    choice=${1}
  fi
	case "${choice}" in
		[Nn]) echo;;
		*) echo
			echo -en "\n${idsCL[Yellow]}Restarting services ... "
			#systemctl restart pve-cluster pvedaemon pvestatd pveproxy pve-ha-lrm pve-ha-crm
			systemctl restart pve-cluster pvedaemon pvestatd pveproxy
			echo -e "${idsCL[Green]}Done${idsCL[Default]}"
			echo -e "\n${idsCL[Green]}This hosts Proxmox services have been restarted${idsCL[Default]}\n"
			[ ${action-x} ] && exit 0 || ENTER2CONTINUE
		;;
	esac
}

SET_VM_SHUTDOWNTIMEOUT(){
  if [ "${1}" == "" ]; then
	  echo -en "${idsCL[LightCyan]}Would you like to set all VM's shutdown timeout to 180secs (Y/n)?${idsCL[Default]} "
	  read -n 1 choice
  else
    choice=${1}
  fi
	case "${choice}" in
		[Nn]) echo;;
		*)
			echo "Updating all VM's shutdown timeout to 180 seconds..."			
			sed -E -i 's/(down=)[0-9]+/\1180/g' /etc/pve/nodes/*/qemu-server/*.conf
			[ ${action-x} ] && exit 0 || ENTER2CONTINUE
		;;
	esac
}

MAINTENANCE_MODE(){
	if ha-manager status | grep $(hostname) | grep "maintenance mode" &> /dev/null; then
		echo -en "${idsCL[LightCyan]}Take the local host out of maintenance mode (Y/n)?${idsCL[Default]} "
	else
		echo -en "${idsCL[LightCyan]}Put the local host into maintenance mode (Y/n)?${idsCL[Default]} "
	fi
	read -n 1 choice
	case "$choice" in
		[Nn]) echo;;
		*) echo
			if ha-manager status | grep $(hostname) | grep "maintenance mode" &> /dev/null; then
				ha-manager crm-command node-maintenance disable $(hostname) &
				echo -e "\n${idsCL[Green]}This host will be taken out of maintenance mode${idsCL[Default]}\n"
			else
				ha-manager crm-command node-maintenance enable $(hostname) &
				echo -e "\n${idsCL[Green]}This host will be entered into maintenance mode${idsCL[Default]}\n"
			fi
			[ ${action-x} ] && exit 0 || ENTER2CONTINUE
		;;
	esac
}

INSTALL_KEEPALIVE() {
	read -n 1 -p "Are you sure you wish to install Keepalive on all Hosts (Y/n)?" choice
	case "$choice" in 
	[Nn]) echo;;
	* )
		echo
		
    set -Eeuo pipefail

    # -----------------------------------------------------------------------------
    # Proxmox cluster Keepalived deployment
    #
    # - Discovers cluster members with `pvecm nodes`
    # - Uses the first non-VIP IPv4 address on the selected interface as each
    #   node's Keepalived unicast source address
    # - Derives VRRP priority from the hostname's trailing numeric suffix
    #     priority = PRIORITY_BASE - trailing_number
    #   so pve1 > pve2 > pve3 when PRIORITY_BASE=200
    # - Deploys a Proxmox-aware health check including cluster quorum
    # - Uses nopreempt so the VIP does not fail back after a recovered node returns
    # - Starts the highest-priority node first so initial ownership is deterministic
    #
    # Run this from any healthy, quorate Proxmox cluster node as root.
    # -----------------------------------------------------------------------------

    VIP_CIDR="10.2.1.10/24"
    INTERFACE="vmbr0"
    PRIORITY_BASE=200
    VRID=51
    ADVERT_INT=1
    ASSUME_YES=0

    SSH_OPTS=(
      -o BatchMode=yes
      -o ConnectTimeout=8
      -o StrictHostKeyChecking=accept-new
    )

    usage() {
      cat <<USAGE
Usage: $0 [options]

Options:
  --vip CIDR              Floating VIP in CIDR form (default: ${VIP_CIDR})
  --interface IFACE       Management interface/bridge (default: ${INTERFACE})
  --priority-base N       Priority calculation base (default: ${PRIORITY_BASE})
                          priority = base - trailing hostname number
  --vrid N                VRRP virtual_router_id, 1-255 (default: ${VRID})
  -y, --yes               Do not prompt before deployment
  -h, --help              Show this help

Examples:
  $0 --vip 10.2.1.10/24 --interface vmbr0
  $0 --vip 192.168.50.10/24 --priority-base 220 --vrid 61 -y
USAGE
    }

    log()  { printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"; }
    warn() { printf '\nWARNING: %s\n' "$*" >&2; }
    die()  { printf '\nERROR: %s\n' "$*" >&2; exit 1; }

    while [[ $# -gt 0 ]]; do
      case "$1" in
        --vip)
          [[ $# -ge 2 ]] || die "--vip requires a value"
          VIP_CIDR="$2"; shift 2 ;;
        --interface)
          [[ $# -ge 2 ]] || die "--interface requires a value"
          INTERFACE="$2"; shift 2 ;;
        --priority-base)
          [[ $# -ge 2 ]] || die "--priority-base requires a value"
          PRIORITY_BASE="$2"; shift 2 ;;
        --vrid)
          [[ $# -ge 2 ]] || die "--vrid requires a value"
          VRID="$2"; shift 2 ;;
        -y|--yes)
          ASSUME_YES=1; shift ;;
        -h|--help)
          usage; exit 0 ;;
        *)
          die "Unknown option: $1" ;;
      esac
    done

    [[ $EUID -eq 0 ]] || die "Run this script as root."
    command -v pvecm >/dev/null 2>&1 || die "pvecm not found. Run this on a Proxmox VE node."
    command -v python3 >/dev/null 2>&1 || die "python3 is required."

    [[ "$INTERFACE" =~ ^[A-Za-z0-9_.:-]+$ ]] || die "Invalid interface name: $INTERFACE"
    [[ "$PRIORITY_BASE" =~ ^[0-9]+$ ]] || die "Priority base must be numeric."
    [[ "$VRID" =~ ^[0-9]+$ ]] || die "VRID must be numeric."
    (( VRID >= 1 && VRID <= 255 )) || die "VRID must be between 1 and 255."

    # Validate VIP/CIDR and derive the bare IP.
    VIP_IP="$(python3 - "$VIP_CIDR" <<'PY'
import ipaddress, sys
try:
    iface = ipaddress.ip_interface(sys.argv[1])
except ValueError as exc:
    print(exc, file=sys.stderr)
    raise SystemExit(1)
if iface.version != 4:
    print("Only IPv4 VIPs are supported by this script.", file=sys.stderr)
    raise SystemExit(1)
print(iface.ip)
PY
    )" || die "Invalid VIP/CIDR: $VIP_CIDR"

    # This check intentionally uses the Proxmox-reported quorum state.
    if ! timeout 5 pvecm status 2>/dev/null | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$'; then
      die "The local node does not currently see a quorate cluster. Refusing deployment."
    fi

    LOCAL_NODE="$(pvecm nodes | awk '$NF == "(local)" {print $3; exit}')"
    [[ -n "$LOCAL_NODE" ]] || LOCAL_NODE="$(hostname -s)"
    DEPLOY_TAG="$(date +%Y%m%d-%H%M%S)-$$"
    TMPDIR="$(mktemp -d)"
    trap 'rm -rf "$TMPDIR"' EXIT

    remote_exec() {
      local node="$1"
      local cmd="$2"

      if [[ "$node" == "$LOCAL_NODE" ]]; then
        bash -lc "$cmd"
      else
        ssh "${SSH_OPTS[@]}" "root@${node}" "$cmd"
      fi
    }

    copy_to_node() {
      local node="$1"
      local src="$2"
      local dst="$3"

      if [[ "$node" == "$LOCAL_NODE" ]]; then
        cp -f "$src" "$dst"
      else
        scp -q "${SSH_OPTS[@]}" "$src" "root@${node}:${dst}"
      fi
    }

    # Discover cluster node names. pvecm nodes emits: Nodeid Votes Name [(local)]
    mapfile -t NODES < <(pvecm nodes | awk '$1 ~ /^(0x)?[0-9A-Fa-f]+$/ && $2 ~ /^[0-9]+$/ {print $3}')
    (( ${#NODES[@]} >= 2 )) || die "Fewer than two cluster nodes were discovered."

    declare -A NODE_IP
    declare -A NODE_PRIORITY
    declare -A PRIORITY_OWNER

    log "Preflight: discovered ${#NODES[@]} cluster nodes."

    # First pass: verify SSH/access, interface addresses, health, suffixes and priorities.
    for node in "${NODES[@]}"; do
      if [[ "$node" != "$LOCAL_NODE" ]]; then
        ssh "${SSH_OPTS[@]}" "root@${node}" true \
          || die "Passwordless root SSH to ${node} failed. No changes have been made."
      fi

      remote_exec "$node" "ip link show '$INTERFACE' >/dev/null 2>&1" \
        || die "Interface ${INTERFACE} does not exist on ${node}."

      # Exclude the VIP so the script remains safe/idempotent if Keepalived is
      # already running and this deployment is being refreshed.
      ip="$(remote_exec "$node" \
        "ip -4 -o addr show dev '$INTERFACE' scope global | awk '{split(\$4,a,\"/\"); print a[1]}' | grep -vx '$VIP_IP' | head -n1")"

      [[ -n "$ip" ]] || die "Could not determine a non-VIP IPv4 address on ${node}:${INTERFACE}."
      [[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || die "Unexpected IPv4 address '${ip}' on ${node}."

      # Verify each management address resides inside the VIP's subnet.
      python3 - "$VIP_CIDR" "$ip" <<'PY' >/dev/null || die "${node} (${ip}) is not in the VIP subnet ${VIP_CIDR}."
import ipaddress, sys
vip = ipaddress.ip_interface(sys.argv[1])
ip = ipaddress.ip_address(sys.argv[2])
raise SystemExit(0 if ip in vip.network else 1)
PY

      [[ "$node" =~ ([0-9]+)$ ]] \
        || die "Hostname '${node}' has no trailing numeric suffix."

      # 10# prevents numbers with leading zeroes from being interpreted as octal.
      suffix=$((10#${BASH_REMATCH[1]}))
      priority=$((PRIORITY_BASE - suffix))

      (( priority >= 1 && priority <= 254 )) \
        || die "Calculated priority ${priority} for ${node} is outside 1..254. Adjust --priority-base."

      if [[ -n "${PRIORITY_OWNER[$priority]:-}" ]]; then
        die "Priority collision: ${node} and ${PRIORITY_OWNER[$priority]} both calculate to ${priority}."
      fi

      # Verify the remote node itself currently sees quorum before touching it.
      remote_exec "$node" \
        "timeout 5 pvecm status 2>/dev/null | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$'" \
        || die "${node} does not currently report Quorate: Yes. No changes have been made."

      NODE_IP["$node"]="$ip"
      NODE_PRIORITY["$node"]="$priority"
      PRIORITY_OWNER["$priority"]="$node"
    done

    # Prevent accidentally assigning a VIP equal to a real node address.
    for node in "${NODES[@]}"; do
      [[ "${NODE_IP[$node]}" != "$VIP_IP" ]] \
        || die "VIP ${VIP_IP} is already the static management address of ${node}."
    done

    # Sort nodes by descending priority. This matters with nopreempt: start the
    # highest-priority node first so it deterministically becomes the initial MASTER.
    mapfile -t SORTED_NODES < <(
      for node in "${NODES[@]}"; do
        printf '%s %s\n' "${NODE_PRIORITY[$node]}" "$node"
      done | sort -nr | awk '{print $2}'
    )

    printf '\nDeployment plan:\n'
    printf '  VIP:        %s\n' "$VIP_CIDR"
    printf '  Interface:  %s\n' "$INTERFACE"
    printf '  VRID:       %s\n' "$VRID"
    printf '  Failback:   disabled (nopreempt)\n\n'
    printf '  %-28s %-16s %-8s\n' "NODE" "MANAGEMENT IP" "PRIORITY"
    printf '  %-28s %-16s %-8s\n' "----------------------------" "----------------" "--------"
    for node in "${SORTED_NODES[@]}"; do
      printf '  %-28s %-16s %-8s\n' "$node" "${NODE_IP[$node]}" "${NODE_PRIORITY[$node]}"
    done

    if (( ASSUME_YES == 0 )); then
      printf '\nThis will install/reconfigure Keepalived on every listed node.\n'
      read -r -p 'Proceed? [y/N] ' answer
      [[ "$answer" =~ ^[Yy]$ ]] || { echo "Aborted."; exit 0; }
    fi

    HEALTH_FILE="$TMPDIR/check-proxmox-keepalived.sh"
    cat > "$HEALTH_FILE" <<'HEALTH'
#!/usr/bin/env bash
set -u

# A node may own the management VIP only if the Proxmox management stack,
# Corosync and cluster quorum are all healthy.

systemctl is-active --quiet pveproxy  || exit 1
systemctl is-active --quiet pvedaemon || exit 1
systemctl is-active --quiet pve-cluster || exit 1
systemctl is-active --quiet corosync || exit 1

# A surviving but non-quorate node is intentionally ineligible for the VIP.
timeout 3 pvecm status 2>/dev/null \
  | grep -Eq '^Quorate:[[:space:]]+Yes[[:space:]]*$' || exit 1

# Verify the actual GUI/API endpoint is answering locally.
curl --silent --show-error --fail --insecure --max-time 3 \
  https://127.0.0.1:8006/ \
  >/dev/null 2>&1 || exit 1

exit 0
HEALTH
    chmod 0750 "$HEALTH_FILE"

    log "Installing Keepalived/curl and staging configuration on all nodes."

    for node in "${NODES[@]}"; do
      log "Preparing ${node}"

      remote_exec "$node" \
        "DEBIAN_FRONTEND=noninteractive apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y keepalived curl >/dev/null"

      remote_health="/tmp/check-proxmox-keepalived.${DEPLOY_TAG}"
      copy_to_node "$node" "$HEALTH_FILE" "$remote_health"
      remote_exec "$node" \
        "install -o root -g root -m 0750 '$remote_health' /usr/local/sbin/check-proxmox-keepalived.sh && rm -f '$remote_health'"

      config_file="$TMPDIR/keepalived-${node}.conf"

      {
        cat <<CONF
# Managed by deploy-proxmox-keepalived.sh
# Deployment: ${DEPLOY_TAG}

global_defs {
    router_id ${node//[^A-Za-z0-9_]/_}
    script_user root
    enable_script_security
}

vrrp_script chk_proxmox {
    script "/usr/local/sbin/check-proxmox-keepalived.sh"
    interval 2
    timeout 4
    fall 3
    rise 3
}

vrrp_instance PROXMOX_VIP {
    state BACKUP
    interface ${INTERFACE}
    virtual_router_id ${VRID}
    priority ${NODE_PRIORITY[$node]}
    advert_int ${ADVERT_INT}
    nopreempt

    unicast_src_ip ${NODE_IP[$node]}

    unicast_peer {
CONF

        for peer in "${NODES[@]}"; do
          [[ "$peer" == "$node" ]] && continue
          printf '        %s\n' "${NODE_IP[$peer]}"
        done

        cat <<CONF
    }

    virtual_ipaddress {
        ${VIP_CIDR} dev ${INTERFACE}
    }

    track_script {
        chk_proxmox
    }
}
CONF
      } > "$config_file"

      remote_conf="/tmp/keepalived.conf.${DEPLOY_TAG}"
      copy_to_node "$node" "$config_file" "$remote_conf"

      # Validate before replacing the live config.
      remote_exec "$node" "keepalived -t -f '$remote_conf'" \
        || die "Keepalived rejected the generated config for ${node}. Existing config was not replaced."

      remote_exec "$node" \
        "if [[ -f /etc/keepalived/keepalived.conf ]]; then cp -a /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.pre-proxmox-vip.${DEPLOY_TAG}; fi; install -o root -g root -m 0644 '$remote_conf' /etc/keepalived/keepalived.conf; rm -f '$remote_conf'; systemctl enable keepalived >/dev/null"

      # Confirm the health script itself is currently passing.
      remote_exec "$node" "/usr/local/sbin/check-proxmox-keepalived.sh" \
        || die "Health check failed on ${node}; Keepalived has not been restarted yet."
    done

    log "All configurations validated. Resetting Keepalived election state."

    # Stop all instances so a rerun cannot preserve an unintended existing master.
    for node in "${NODES[@]}"; do
      remote_exec "$node" "systemctl stop keepalived" || true
    done

    MASTER_NODE="${SORTED_NODES[0]}"
    MASTER_IP="${NODE_IP[$MASTER_NODE]}"

    log "Starting highest-priority node first: ${MASTER_NODE} (${NODE_PRIORITY[$MASTER_NODE]})."
    remote_exec "$MASTER_NODE" "systemctl start keepalived"

    # Wait until the preferred initial node actually owns the VIP before starting
    # nopreempt backups. This prevents a lower-priority node from winning first.
    master_ready=0
    for _ in {1..12}; do
      if remote_exec "$MASTER_NODE" \
          "ip -4 -o addr show dev '$INTERFACE' | awk '{print \$4}' | cut -d/ -f1 | grep -Fxq '$VIP_IP'"; then
        master_ready=1
        break
      fi
      sleep 1
    done

    (( master_ready == 1 )) \
      || die "${MASTER_NODE} did not acquire VIP ${VIP_IP}. Keepalived remains stopped on the other nodes. Check: journalctl -u keepalived"

    for node in "${SORTED_NODES[@]:1}"; do
      log "Starting backup node ${node}."
      remote_exec "$node" "systemctl start keepalived"
    done

    log "Verifying VIP ownership and service state."

    owners=()
    for node in "${NODES[@]}"; do
      remote_exec "$node" "systemctl is-active --quiet keepalived" \
        || die "Keepalived is not active on ${node}."

      if remote_exec "$node" \
          "ip -4 -o addr show dev '$INTERFACE' | awk '{print \$4}' | cut -d/ -f1 | grep -Fxq '$VIP_IP'"; then
        owners+=("$node")
      fi
    done

    (( ${#owners[@]} == 1 )) \
      || die "Expected exactly one VIP owner, found ${#owners[@]}: ${owners[*]:-none}"

    # Verify the VIP itself answers the Proxmox API from the deployment node.
    if ! curl --silent --show-error --fail --insecure --max-time 5 \
        "https://${VIP_IP}:8006/" >/dev/null 2>&1; then
      warn "VIP ${VIP_IP} is assigned to ${owners[0]}, but the API test through the VIP failed."
      warn "Check routing/firewall rules and whether pveproxy has been restricted to a specific LISTEN_IP."
    else
      log "VIP API test succeeded."
    fi

    cat <<RESULT

Deployment complete.

VIP:           ${VIP_IP}
Current owner: ${owners[0]}
Priority:      ${NODE_PRIORITY[${owners[0]}]}
URL:           https://${VIP_IP}:8006/

Health requirements for VIP ownership:
  - pveproxy active
  - pvedaemon active
  - pve-cluster active
  - corosync active
  - pvecm reports Quorate: Yes
  - local Proxmox API responds on 127.0.0.1:8006

Failback is disabled with nopreempt. After a failure, the surviving MASTER keeps
ownership when a higher-priority node returns.

Useful commands:
  ip -br addr show ${INTERFACE}
  systemctl status keepalived
  journalctl -u keepalived -f
  /usr/local/sbin/check-proxmox-keepalived.sh; echo \$?
RESULT
    
		echo -e "\n${idsCL[Green]}Keepalive has been installed${idsCL[Default]}"
		[ ${action-x} ] && exit 0 || ENTER2CONTINUE
	esac
}



MAIN_MENU() {
	echo -en "${idsCL[LightCyan]}Pulling host info ... "
  CRES=$(pvesh get /cluster/resources)
  DPL=$(dpkg -l)
	echo -e "${idsCL[Green]}Done${idsCL[Default]}"
  
	while :
	do
		clear
		echo
		echo -e " ${idsCL[Green]}TA-Proxmenu - Proxmox Setup Scripts${idsCL[Default]}                           ${idsCL[Default]}${VERS}"
		echo -e "${idsCL[Green]}---------------------------------------------------------------------------${idsCL[Default]}"
		echo -e " Hostname:    ${idsCL[Cyan]}$(hostname -s)${idsCL[Default]}"
		echo -e " IP Address:  ${idsCL[Cyan]}${RNIP}${idsCL[Default]}"
		echo -e "${idsCL[Green]}---------------------------------------------------------------------------${idsCL[Default]}"
		echo
        
		echo -en "${idsCL[White]} [${idsCL[LightYellow]}0${idsCL[Default]}] ${idsCL[White]}Run Post-Install Script${idsCL[Default]}"
		[ -f /opt/.PROXMENUX_POST_INSTALL ] && echo -e "${idsCL[Cyan]} - Has been ran prevously${idsCL[Default]}" || echo
		echo -e "${idsCL[White]} [${idsCL[LightYellow]}1${idsCL[Default]}] ${idsCL[White]}Detect CPU-Arch for Live Migrations${idsCL[Default]}"
		if ! echo ${CRES} | grep -i pulse &> /dev/null ; then
			echo -e "${idsCL[White]} [${idsCL[LightYellow]}2${idsCL[Default]}] ${idsCL[White]}Install Pulse Monitoring${idsCL[Default]}"
		else
			echo -e "${idsCL[DarkGray]} [2] Pulse Monitoring is already installed${idsCL[Default]}"
		fi
    if [ -f ${DLDIR}/${VIRTIO_FILE} ]; then
			echo -e "${idsCL[DarkGray]} [3] Current VirtIO drivers already downloaded to 'local' on this host${idsCL[Default]}"
    elif [ -f ${DLDIR}/virtio*.iso ]; then
      echo -e "${idsCL[White]} [${idsCL[LightYellow]}3${idsCL[Default]}] ${idsCL[LightGreen]}**${idsCL[White]}Download the available updated Win-VirtIO drivers to 'local' on this host${idsCL[Default]}"
    else
      echo -e "${idsCL[White]} [${idsCL[LightYellow]}3${idsCL[Default]}] ${idsCL[White]}Download the current Win-VirtIO drivers to 'local' on this host${idsCL[Default]}"
    fi
		if ! command -v glances &> /dev/null; then
			echo -e "${idsCL[White]} [${idsCL[LightYellow]}4${idsCL[Default]}] ${idsCL[White]}Install Glances (CLI Monitor)${idsCL[Default]}"
		else
			echo -e "${idsCL[DarkGray]} [4] Glances is already installed${idsCL[Default]}"
		fi
		if [ "$(echo ${DPL} | awk '/srvadmin-all/ {print }'|wc -l)" -eq 0 ]; then
			echo -e "${idsCL[White]} [${idsCL[LightYellow]}5${idsCL[Default]}] ${idsCL[White]}Install Dell OpenManage Server Administrator${idsCL[Default]}"
		else
			echo -e "${idsCL[DarkGray]} [5] Dell OMSA is already installed - ${idsCL[Cyan]}https://${RNIP}:1311"
		fi		
		echo
		if [ "$(systemctl is-active ITSPlatform)" != "active" ]; then
			echo -e "${idsCL[White]} [${idsCL[LightYellow]}6${idsCL[Default]}] ${idsCL[White]}Install ConnectWise RMM Agent${idsCL[Default]}"
		else
			echo -e "${idsCL[DarkGray]} [6] ConnectWise RMM Agent is already installed${idsCL[Default]}"
		fi	
		if [ "$(echo ${DPL} | awk '/cyberprotect/ {print }'|wc -l)" -eq 0 ]; then
			echo -e "${idsCL[White]} [${idsCL[LightYellow]}7${idsCL[Default]}] ${idsCL[White]}Install Acronis Backup Agent${idsCL[Default]}"
		else
			echo -e "${idsCL[DarkGray]} [7] Acronis Backup is already installed${idsCL[Default]}"
		fi
		if [ "$(echo ${DPL} | awk '/sentinelagent/ {print }'|wc -l)" -eq 0 ]; then
			echo -e "${idsCL[White]} [${idsCL[LightYellow]}8${idsCL[Default]}] ${idsCL[White]}Install SentinelOne Agent (v25_4_2_21)${idsCL[Default]}"
		else
			echo -e "${idsCL[DarkGray]} [8] SentinelOne is already installed${idsCL[Default]}"
		fi
		echo
		if [ "$(systemctl is-active connectwise*)" != "active" ]; then
			echo -e "${idsCL[White]} [${idsCL[LightYellow]}S${idsCL[Default]}] ${idsCL[White]}Install ScreenConnect Agent${idsCL[Default]}"
		else
			echo -e "${idsCL[DarkGray]} [S] ScreenConnect is already installed${idsCL[Default]}"
		fi
		echo
		if ha-manager status | grep $(hostname) | grep "maintenance mode" &> /dev/null; then
			echo -e "${idsCL[White]} [${idsCL[LightYellow]}M${idsCL[Default]}] ${idsCL[White]}Take Host out of Maintenance Mode${idsCL[Default]}"
		else
			echo -e "${idsCL[White]} [${idsCL[LightYellow]}M${idsCL[Default]}] ${idsCL[White]}Put Host into Maintenance Mode${idsCL[Default]}"
		fi
		echo -e "${idsCL[White]} [${idsCL[LightYellow]}R${idsCL[Default]}] ${idsCL[White]}Restart Proxmox Services${idsCL[Default]}"
		echo
		echo -e "${idsCL[White]} [${idsCL[LightYellow]}T${idsCL[Default]}] ${idsCL[White]}FIX: Set ALL VMs shutdown-timeout to 180secs${idsCL[Default]}"
		echo
		echo -e "${idsCL[White]} [${idsCL[LightYellow]}Q${idsCL[Default]}] ${idsCL[White]}Quit${idsCL[Default]}"
		echo
		echo
		echo -e -n "${idsCL[Yellow]}Enter ${idsCL[LightYellow]}option${idsCL[Yellow]} from above:${idsCL[Default]} "
		read -n 1 opt
		echo
		case $opt in
		  [0]) PROXMENUX_POST_INSTALL;;
		  [1]) DETECT_CPU;;
		  [2]) INSTALL_PULSE;;
		  [3]) DOWNLOAD_VIRTIO;;
		  [4]) INSTALL_GLANCES;;
		  [5]) INSTALL_OMSA;;
		  [6]) INSTALL_RMM;;
		  [7]) INSTALL_ACRONIS;;
		  [8]) INSTALL_S1;;
		  
		  [Ss]) INSTALL_SCREENCONNECT;;
		  [Mm]) MAINTENANCE_MODE;;
		  [Rr]) RESTART_PVE_SERVICES;;
		  [Tt]) SET_VM_SHUTDOWNTIMEOUT;;
		  [Qq]) EXIT1; exit 0;;
			 *)	echo -e "Thats an invaild option,\nplease select a valid option only."; sleep 1;;
		esac
	done
}


if [ ${action-x} ]; then
	case $action in
		pulse) INSTALL_PULSE;;
		rmm) INSTALL_RMM;;
		omsa) INSTALL_OMSA;;
		glances) INSTALL_GLANCES;;
		acronis) INSTALL_ACRONIS;;
		proxmenux) [ ! -f /usr/local/bin/menu ] && INSTALL_PROXMENUX || /usr/local/bin/menu;;
		screenconnect) INSTALL_SCREENCONNECT;;
		restart) RESTART_PVE_SERVICES ${2};;
		cpu) DETECT_CPU;;
		mm) MAINTENANCE_MODE;;
		timeout) SET_VM_SHUTDOWNTIMEOUT ${2};;
    *) MAIN_MENU;;
	esac
else
	MAIN_MENU
fi

exit 0