diff --git a/defaults.inc b/defaults.inc index 6e4040c..cf839e4 100755 --- a/defaults.inc +++ b/defaults.inc @@ -3,7 +3,7 @@ action="${1:-}" FOLDER='/opt/idssys/ta-proxmenu' -VERS='2026.7.25-11' +VERS='2026.7.25-12' noupdate=' ' diff --git a/inc/evacuate-proxmox-node.sh b/inc/evacuate-proxmox-node.sh new file mode 100755 index 0000000..516dfba --- /dev/null +++ b/inc/evacuate-proxmox-node.sh @@ -0,0 +1,371 @@ +#!/usr/bin/env bash +set -u + +# Evacuate non-HA guests after the local Proxmox node enters HA maintenance. +# Guests using shared storage are migrated to one operator-selected node. +# Guests with local storage remain on this node and are gracefully shut down. + +HA_WAIT_SECONDS=300 +SHUTDOWN_TIMEOUT=180 +LOCAL_NODE="$(hostname -s)" + +log() { + printf '\n[%s] %s\n' "$(date '+%F %T')" "$*" +} + +warn() { + printf '\nWARNING: %s\n' "$*" >&2 +} + +die() { + printf '\nERROR: %s\n' "$*" >&2 + exit 1 +} + +command -v pvesh >/dev/null 2>&1 || die "pvesh is required." +command -v ha-manager >/dev/null 2>&1 || die "ha-manager is required." +command -v python3 >/dev/null 2>&1 || die "python3 is required." + +pvesh get /cluster/resources --type vm --output-format json >/dev/null 2>&1 || + die "Could not read cluster guest resources." +pvesh get /cluster/ha/resources --output-format json >/dev/null 2>&1 || + die "Could not read HA resources." +pvesh get /nodes --output-format json >/dev/null 2>&1 || + die "Could not read cluster node status." +pvesh get /storage --output-format json >/dev/null 2>&1 || + die "Could not read cluster storage configuration." + +get_local_guests() { + pvesh get /cluster/resources --type vm --output-format json 2>/dev/null | + python3 -c ' +import json +import sys + +node = sys.argv[1] +for guest in json.load(sys.stdin): + if guest.get("node") != node or guest.get("type") not in ("qemu", "lxc"): + continue + print( + guest.get("vmid", ""), + guest.get("type", ""), + guest.get("status", "unknown"), + str(guest.get("name", "")).replace("\x1f", " "), + sep="\x1f", + ) +' "$LOCAL_NODE" +} + +get_ha_guest_ids() { + pvesh get /cluster/ha/resources --output-format json 2>/dev/null | + python3 -c ' +import json +import re +import sys + +for resource in json.load(sys.stdin): + sid = str(resource.get("sid") or resource.get("service") or "") + match = re.fullmatch(r"(?:vm|ct):(\d+)", sid) + if match: + print(match.group(1)) +' +} + +get_online_nodes() { + pvesh get /nodes --output-format json 2>/dev/null | + python3 -c ' +import json +import sys + +local_node = sys.argv[1] +for node in json.load(sys.stdin): + name = str(node.get("node", "")) + if name and name != local_node and node.get("status") == "online": + print(name) +' "$LOCAL_NODE" +} + +get_shared_storages() { + pvesh get /storage --output-format json 2>/dev/null | + python3 -c ' +import json +import sys + +for storage in json.load(sys.stdin): + if storage.get("shared"): + print(storage.get("storage", "")) +' +} + +guest_storage_scope() { + local guest_type="$1" + local vmid="$2" + local shared_csv="$3" + + pvesh get "/nodes/${LOCAL_NODE}/${guest_type}/${vmid}/config" \ + --output-format json 2>/dev/null | + python3 -c ' +import json +import re +import sys + +guest_type, shared_csv = sys.argv[1:3] +shared = {item for item in shared_csv.split(",") if item} +config = json.load(sys.stdin) + +if guest_type == "qemu": + disk_key = re.compile( + r"^(?:ide|sata|scsi|virtio|efidisk|tpmstate|unused)\d+$" + ) +else: + disk_key = re.compile(r"^(?:rootfs|mp\d+|unused\d+)$") + +local_reasons = [] +for key, raw_value in config.items(): + if not disk_key.match(key) or not isinstance(raw_value, str): + continue + + volume = raw_value.split(",", 1)[0] + if volume in ("none", "cdrom") or volume.startswith("none,"): + continue + if volume.startswith("/") or ":" not in volume: + local_reasons.append(f"{key}={volume}") + continue + + storage = volume.split(":", 1)[0] + if storage not in shared: + local_reasons.append(f"{key}={storage}") + +if local_reasons: + print("local\x1f" + ", ".join(local_reasons)) +else: + print("shared\x1f") +' "$guest_type" "$shared_csv" +} + +select_target_node() { + local -a nodes + local choice + local index + + mapfile -t nodes < <(get_online_nodes) + (( ${#nodes[@]} > 0 )) || die "No other online cluster node is available." + + printf '\nAvailable migration targets:\n\n' + for index in "${!nodes[@]}"; do + printf ' %d) %s\n' "$((index + 1))" "${nodes[$index]}" + done + + while true; do + printf '\n' + read -r -p "Select migration target [1-${#nodes[@]}]: " choice + if [[ "$choice" =~ ^[0-9]+$ ]] && + (( choice >= 1 && choice <= ${#nodes[@]} )); then + TARGET_NODE="${nodes[$((choice - 1))]}" + return + fi + printf 'Invalid selection.\n' >&2 + done +} + +wait_for_ha_evacuation() { + local deadline=$((SECONDS + HA_WAIT_SECONDS)) + local -a local_guests + local -a ha_ids + local -a remaining + local guest + local ha_id + + log "Waiting for HA-managed guests to leave ${LOCAL_NODE}." + + while true; do + mapfile -t local_guests < <(get_local_guests) + mapfile -t ha_ids < <(get_ha_guest_ids) + remaining=() + + for guest in "${local_guests[@]}"; do + for ha_id in "${ha_ids[@]}"; do + if [[ "${guest%%$'\x1f'*}" == "$ha_id" ]]; then + remaining+=("$guest") + break + fi + done + done + + (( ${#remaining[@]} == 0 )) && return + + if (( SECONDS >= deadline )); then + warn "HA evacuation did not finish within ${HA_WAIT_SECONDS} seconds." + printf 'HA-managed guests still assigned to %s:\n' "$LOCAL_NODE" >&2 + printf ' %s\n' "${remaining[@]}" >&2 + return 1 + fi + + printf '\r Waiting: %d HA guest(s) remain... ' "${#remaining[@]}" + sleep 5 + done +} + +migrate_guest() { + local vmid="$1" + local guest_type="$2" + local status="$3" + + if [[ "$guest_type" == "qemu" ]]; then + if [[ "$status" == "running" ]]; then + qm migrate "$vmid" "$TARGET_NODE" --online + else + qm migrate "$vmid" "$TARGET_NODE" + fi + else + if [[ "$status" == "running" ]]; then + pct migrate "$vmid" "$TARGET_NODE" --restart 1 \ + --timeout "$SHUTDOWN_TIMEOUT" + else + pct migrate "$vmid" "$TARGET_NODE" + fi + fi +} + +shutdown_guest() { + local vmid="$1" + local guest_type="$2" + + if [[ "$guest_type" == "qemu" ]]; then + qm shutdown "$vmid" --timeout "$SHUTDOWN_TIMEOUT" + else + pct shutdown "$vmid" --timeout "$SHUTDOWN_TIMEOUT" + fi +} + +guest_status() { + local vmid="$1" + local guest_type="$2" + + if [[ "$guest_type" == "qemu" ]]; then + qm status "$vmid" 2>/dev/null | awk '{ print $2 }' + else + pct status "$vmid" 2>/dev/null | awk '{ print $2 }' + fi +} + +if ! ha-manager status | + grep -F "$LOCAL_NODE" | + grep -q "maintenance mode"; then + die "${LOCAL_NODE} is not in HA maintenance mode." +fi + +wait_for_ha_evacuation || + die "Resolve the remaining HA guests before continuing the evacuation." + +select_target_node + +mapfile -t shared_storages < <(get_shared_storages) +shared_csv="$(IFS=,; echo "${shared_storages[*]}")" + +mapfile -t guests < <(get_local_guests) +if (( ${#guests[@]} == 0 )); then + log "No guests remain on ${LOCAL_NODE}." + exit 0 +fi + +declare -a shared_guests=() +declare -a local_guests=() + +for guest in "${guests[@]}"; do + IFS=$'\x1f' read -r vmid guest_type status name <<< "$guest" + storage_result="$(guest_storage_scope "$guest_type" "$vmid" "$shared_csv")" || + die "Could not inspect storage for ${guest_type} ${vmid}." + IFS=$'\x1f' read -r scope reason <<< "$storage_result" + + if [[ "$scope" == "shared" ]]; then + shared_guests+=("$guest") + else + local_guests+=("${guest}"$'\x1f'"${reason:-local storage}") + fi +done + +printf '\nEvacuation plan for %s:\n' "$LOCAL_NODE" +printf ' Migration target: %s\n' "$TARGET_NODE" +printf ' Shared-storage guests to migrate: %d\n' "${#shared_guests[@]}" +printf ' Local-storage guests to retain: %d\n' "${#local_guests[@]}" + +if (( ${#shared_guests[@]} > 0 )); then + printf '\nShared-storage guests:\n' + for guest in "${shared_guests[@]}"; do + IFS=$'\x1f' read -r vmid guest_type status name <<< "$guest" + printf ' %-6s %-5s %-8s %s\n' "$vmid" "$guest_type" "$status" "$name" + done +fi + +if (( ${#local_guests[@]} > 0 )); then + printf '\nLocal-storage guests (will not migrate):\n' + for guest in "${local_guests[@]}"; do + IFS=$'\x1f' read -r vmid guest_type status name reason <<< "$guest" + printf ' %-6s %-5s %-8s %-24s %s\n' \ + "$vmid" "$guest_type" "$status" "$name" "$reason" + done +fi + +printf '\n' +read -r -p "Proceed with shared-storage guest migration? [y/N] " answer +[[ "$answer" =~ ^[Yy]$ ]] || { echo "Evacuation cancelled."; exit 0; } + +declare -a migration_failures=() +for guest in "${shared_guests[@]}"; do + IFS=$'\x1f' read -r vmid guest_type status name <<< "$guest" + log "Migrating ${guest_type} ${vmid} (${name:-unnamed}) to ${TARGET_NODE}." + if ! migrate_guest "$vmid" "$guest_type" "$status"; then + warn "Migration failed for ${guest_type} ${vmid}." + migration_failures+=("$guest") + fi +done + +declare -a running_local_guests=() +for guest in "${local_guests[@]}"; do + IFS=$'\x1f' read -r vmid guest_type status name reason <<< "$guest" + status="$(guest_status "$vmid" "$guest_type")" + if [[ "$status" == "running" ]]; then + running_local_guests+=( + "${vmid}"$'\x1f'"${guest_type}"$'\x1f'"${status}"$'\x1f'"${name}"$'\x1f'"${reason}" + ) + fi +done + +if (( ${#running_local_guests[@]} > 0 )); then + printf '\nThe following local-storage guests remain running:\n' + for guest in "${running_local_guests[@]}"; do + IFS=$'\x1f' read -r vmid guest_type status name reason <<< "$guest" + printf ' %-6s %-5s %-24s %s\n' "$vmid" "$guest_type" "$name" "$reason" + done + + printf '\n' + read -r -p "Gracefully shut down these local-storage guests? [y/N] " answer + if [[ "$answer" =~ ^[Yy]$ ]]; then + for guest in "${running_local_guests[@]}"; do + IFS=$'\x1f' read -r vmid guest_type status name reason <<< "$guest" + log "Shutting down ${guest_type} ${vmid} (${name:-unnamed})." + shutdown_guest "$vmid" "$guest_type" || + warn "Graceful shutdown failed for ${guest_type} ${vmid}; it was not force-stopped." + done + else + warn "Local-storage guests were left running." + fi +fi + +mapfile -t final_guests < <(get_local_guests) + +printf '\nEvacuation summary:\n' +printf ' Migration failures: %d\n' "${#migration_failures[@]}" +printf ' Guests still assigned to %s: %d\n' "$LOCAL_NODE" "${#final_guests[@]}" + +if (( ${#migration_failures[@]} > 0 )); then + printf '\nFailed migrations (left unchanged; not shut down automatically):\n' + printf ' %s\n' "${migration_failures[@]}" +fi + +if (( ${#final_guests[@]} > 0 )); then + printf '\nGuests still assigned to %s:\n' "$LOCAL_NODE" + printf ' %s\n' "${final_guests[@]}" +fi + +printf '\nNo guest was force-stopped.\n' diff --git a/proxmenu-scripts.sh b/proxmenu-scripts.sh index 1afe5f9..52121bb 100755 --- a/proxmenu-scripts.sh +++ b/proxmenu-scripts.sh @@ -273,7 +273,10 @@ SET_VM_SHUTDOWNTIMEOUT(){ } MAINTENANCE_MODE(){ - if ha-manager status | grep $(hostname) | grep "maintenance mode" &> /dev/null; then + local local_node + local_node="$(hostname -s)" + + if ha-manager status | grep -F "$local_node" | grep -q "maintenance mode"; then echo -en "${idsCL[LightCyan]}Take the local host out of maintenance mode (Y/n)?${idsCL[Default]} " else echo -en "${idsCL[LightCyan]}Put the local host into maintenance mode (Y/n)?${idsCL[Default]} " @@ -282,12 +285,13 @@ MAINTENANCE_MODE(){ case "$choice" in [Nn]) echo;; *) echo - if ha-manager status | grep $(hostname) | grep "maintenance mode" &> /dev/null; then - ha-manager crm-command node-maintenance disable $(hostname) & + if ha-manager status | grep -F "$local_node" | grep -q "maintenance mode"; then + ha-manager crm-command node-maintenance disable "$local_node" echo -e "\n${idsCL[Green]}This host will be taken out of maintenance mode${idsCL[Default]}\n" else - ha-manager crm-command node-maintenance enable $(hostname) & + ha-manager crm-command node-maintenance enable "$local_node" echo -e "\n${idsCL[Green]}This host will be entered into maintenance mode${idsCL[Default]}\n" + bash /opt/idssys/ta-proxmenu/inc/evacuate-proxmox-node.sh fi FINISH_ACTION ;; @@ -375,7 +379,7 @@ MAIN_MENU() { echo -e "${idsCL[DarkGray]} [S] ScreenConnect is already installed${idsCL[Default]}" fi echo - if ha-manager status | grep $(hostname) | grep "maintenance mode" &> /dev/null; then + if ha-manager status | grep -F "$(hostname -s)" | grep -q "maintenance mode"; then echo -e "${idsCL[White]} [${idsCL[LightYellow]}M${idsCL[Default]}] ${idsCL[White]}Take Host out of Maintenance Mode${idsCL[Default]}" else echo -e "${idsCL[White]} [${idsCL[LightYellow]}M${idsCL[Default]}] ${idsCL[White]}Put Host into Maintenance Mode${idsCL[Default]}" @@ -437,4 +441,3 @@ else fi exit 0 -