TA-ProxMenu/inc/evacuate-proxmox-node.sh

#!/usr/bin/env bash
set -u -o pipefail

# Evacuate non-HA guests after the local Proxmox node enters HA maintenance.
# HA-managed guests remain under Proxmox HA control. Non-HA guests using
# shared storage are migrated, while guests using local storage are shut down.

HA_WAIT_SECONDS=300
MAINTENANCE_WAIT_SECONDS=60
MAX_PARALLEL_MIGRATIONS=3
SHUTDOWN_TIMEOUT=180
LOCAL_NODE="$(hostname -s)"
MIGRATION_LOG_DIR=''
PREFERRED_TARGET=''
HA_RULES_FILE="${HA_RULES_FILE:-/etc/pve/ha/rules.cfg}"

declare -a MIGRATION_NODES=()
declare -A NODE_STORAGE_CACHE=()

log() {
	printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"
}

warn() {
	printf '\nWARNING: %s\n' "$*" >&2
}

die() {
	printf '\nERROR: %s\n' "$*" >&2
	exit 1
}

load_lines() {
	local destination_name="$1"
	local output
	local -n destination_ref="$destination_name"
	shift

	output="$("$@")" || return 1
	destination_ref=()
	# shellcheck disable=SC2034 # mapfile writes through the nameref.
	[[ -z "$output" ]] || mapfile -t destination_ref <<< "$output"
}

cleanup() {
	if [[ "$MIGRATION_LOG_DIR" == /tmp/ta-proxmenu-evacuation.* &&
		-d "$MIGRATION_LOG_DIR" ]]; then
		rm -rf -- "$MIGRATION_LOG_DIR"
	fi
}

get_local_guests() {
	pvesh get /cluster/resources --type vm --output-format json 2>/dev/null |
		python3 -c '
import json
import sys

node = sys.argv[1]
for guest in json.load(sys.stdin):
    if guest.get("node") != node or guest.get("type") not in ("qemu", "lxc"):
        continue
    print(
        guest.get("vmid", ""),
        guest.get("type", ""),
        guest.get("status", "unknown"),
        str(guest.get("name", "")).replace("\x1f", " "),
        int(guest.get("maxmem") or 0),
        sep="\x1f",
    )
' "$LOCAL_NODE"
}

get_ha_guest_ids() {
	pvesh get /cluster/ha/resources --output-format json 2>/dev/null |
		python3 -c '
import json
import re
import sys

for resource in json.load(sys.stdin):
    sid = str(resource.get("sid") or resource.get("service") or "")
    match = re.fullmatch(r"(?:vm|ct):(\d+)", sid)
    if match:
        print(match.group(1))
'
}

guest_is_ha_managed() {
	local vmid="$1"
	local ha_id

	for ha_id in "${CURRENT_HA_IDS[@]:-}"; do
		[[ "$ha_id" == "$vmid" ]] && return 0
	done
	return 1
}

get_online_nodes() {
	pvesh get /nodes --output-format json 2>/dev/null |
		python3 -c '
import json
import sys

local_node = sys.argv[1]
for node in json.load(sys.stdin):
    name = str(node.get("node", ""))
    if name and name != local_node and node.get("status") == "online":
        print(name)
' "$LOCAL_NODE"
}

node_in_maintenance() {
	local node="$1"

	ha-manager status 2>/dev/null |
		grep -F "lrm ${node} " |
		grep -q "maintenance mode"
}

get_shared_storages() {
	pvesh get /storage --output-format json 2>/dev/null |
		python3 -c '
import json
import sys

for storage in json.load(sys.stdin):
    if storage.get("shared"):
        print(storage.get("storage", ""))
'
}

get_node_active_storages() {
	local node="$1"

	pvesh get "/nodes/${node}/storage" --output-format json 2>/dev/null |
		python3 -c '
import json
import sys

for storage in json.load(sys.stdin):
    active = storage.get("active")
    enabled = storage.get("enabled", 1)
    if active in (1, True, "1") and enabled not in (0, False, "0"):
        storage_id = str(storage.get("storage", ""))
        if storage_id:
            print(storage_id)
'
}

refresh_migration_nodes() {
	local ha_status
	local node
	local online_output
	local storage_csv
	local -a online_nodes=()
	local -a usable_nodes=()

	online_output="$(get_online_nodes)" || {
		warn "Could not refresh online cluster nodes."
		return 1
	}
	[[ -z "$online_output" ]] ||
		mapfile -t online_nodes <<< "$online_output"

	ha_status="$(ha-manager status 2>/dev/null)" || {
		warn "Could not refresh HA node status."
		return 1
	}
	NODE_STORAGE_CACHE=()

	for node in "${online_nodes[@]}"; do
		if grep -F "lrm ${node} " <<< "$ha_status" |
			grep -q "maintenance mode"; then
			continue
		fi
		storage_csv="$(get_node_active_storages "$node" | paste -sd, -)" || {
			warn "Could not read storage status from ${node}; it will not be used."
			continue
		}
		NODE_STORAGE_CACHE["$node"]="$storage_csv"
		usable_nodes+=("$node")
	done
	MIGRATION_NODES=("${usable_nodes[@]}")
}

node_has_required_storages() {
	local node="$1"
	local required_csv="$2"
	local available_csv="${NODE_STORAGE_CACHE[$node]-}"
	local storage
	local -a required_storages=()

	[[ -n "$required_csv" ]] || return 0
	IFS=',' read -r -a required_storages <<< "$required_csv"
	for storage in "${required_storages[@]}"; do
		[[ ",${available_csv}," == *",${storage},"* ]] || return 1
	done
}

guest_storage_scope() {
	local guest_type="$1"
	local vmid="$2"
	local shared_csv="$3"

	pvesh get "/nodes/${LOCAL_NODE}/${guest_type}/${vmid}/config" \
		--output-format json 2>/dev/null |
		python3 -c '
import json
import re
import sys

guest_type, shared_csv = sys.argv[1:3]
shared = {item for item in shared_csv.split(",") if item}
config = json.load(sys.stdin)

if guest_type == "qemu":
    disk_key = re.compile(
        r"^(?:ide|sata|scsi|virtio|efidisk|tpmstate|unused)\d+$"
    )
else:
    disk_key = re.compile(r"^(?:rootfs|mp\d+|unused\d+)$")

local_reasons = []
required_storages = set()
for key, raw_value in config.items():
    if not disk_key.match(key) or not isinstance(raw_value, str):
        continue

    volume = raw_value.split(",", 1)[0]
    if volume in ("none", "cdrom") or volume.startswith("none,"):
        continue
    if volume.startswith("/") or ":" not in volume:
        local_reasons.append(f"{key}={volume}")
        continue

    storage = volume.split(":", 1)[0]
    required_storages.add(storage)
    if storage not in shared:
        local_reasons.append(f"{key}={storage}")

scope = "local" if local_reasons else "shared"
print(
    scope,
    ", ".join(local_reasons),
    ",".join(sorted(required_storages)),
    sep="\x1f",
)
' "$guest_type" "$shared_csv"
}

get_guest_node_affinity() {
	local guest_type="$1"
	local vmid="$2"
	local sid_type='vm'

	[[ "$guest_type" == "lxc" ]] && sid_type='ct'
	python3 -c '
import os
import sys

sid = sys.argv[1]
path = sys.argv[2]
if not os.path.exists(path):
    print("none", "0", "", sep="\x1f")
    raise SystemExit

rules = []
current = None
with open(path, encoding="utf-8") as handle:
    for raw_line in handle:
        line = raw_line.rstrip()
        if not line or line.lstrip().startswith("#"):
            continue
        if not line[0].isspace() and ":" in line:
            rule_type, rule_id = line.split(":", 1)
            current = {
                "type": rule_type.strip(),
                "id": rule_id.strip(),
            }
            rules.append(current)
            continue
        if current is not None and line[0].isspace():
            key_value = line.strip().split(None, 1)
            if len(key_value) == 2:
                current[key_value[0]] = key_value[1].strip()

for rule in rules:
    if rule.get("type") != "node-affinity":
        continue
    if rule.get("disable", "0") in ("1", "yes", "true", "on"):
        continue
    resources = {
        item.strip()
        for item in rule.get("resources", "").replace(";", ",").split(",")
        if item.strip()
    }
    if sid not in resources:
        continue

    nodes = []
    for position, item in enumerate(rule.get("nodes", "").split(",")):
        item = item.strip()
        if not item:
            continue
        node = item
        priority = 0
        if ":" in item:
            possible_node, possible_priority = item.rsplit(":", 1)
            try:
                priority = int(possible_priority)
                node = possible_node
            except ValueError:
                pass
        nodes.append((node, priority, position))
    nodes.sort(key=lambda value: (-value[1], value[2]))
    strict = "1" if rule.get("strict", "0") in ("1", "yes", "true", "on") else "0"
    print(rule.get("id", "node-affinity"), strict, ",".join(n[0] for n in nodes), sep="\x1f")
    raise SystemExit

print("none", "0", "", sep="\x1f")
' "${sid_type}:${vmid}" "$HA_RULES_FILE"
}

select_target_node() {
	local choice
	local index

	(( ${#MIGRATION_NODES[@]} > 0 )) ||
		die "No other online, non-maintenance cluster node is available."

	printf '\nAvailable preferred migration targets:\n\n'
	for index in "${!MIGRATION_NODES[@]}"; do
		printf '  %d) %s\n' "$((index + 1))" "${MIGRATION_NODES[$index]}"
	done

	while true; do
		printf '\n'
		read -r -p "Select preferred migration target [1-${#MIGRATION_NODES[@]}]: " choice
		if [[ "$choice" =~ ^[0-9]+$ ]] &&
			(( choice >= 1 && choice <= ${#MIGRATION_NODES[@]} )); then
			PREFERRED_TARGET="${MIGRATION_NODES[$((choice - 1))]}"
			return
		fi
		printf 'Invalid selection.\n' >&2
	done
}

CHOSEN_DESTINATION=''
CHOSEN_NOTE=''

choose_destination() {
	local required_csv="$1"
	local policy_nodes_csv="$2"
	local policy_strict="$3"
	local node
	local policy_node
	local -a storage_candidates=()
	local -a policy_nodes=()

	CHOSEN_DESTINATION=''
	CHOSEN_NOTE=''

	for node in "${MIGRATION_NODES[@]}"; do
		if node_has_required_storages "$node" "$required_csv"; then
			storage_candidates+=("$node")
		fi
	done
	(( ${#storage_candidates[@]} > 0 )) || return 1

	IFS=',' read -r -a policy_nodes <<< "$policy_nodes_csv"
	if [[ -z "$policy_nodes_csv" ]]; then
		for node in "${storage_candidates[@]}"; do
			if [[ "$node" == "$PREFERRED_TARGET" ]]; then
				CHOSEN_DESTINATION="$node"
				return 0
			fi
		done
		CHOSEN_DESTINATION="${storage_candidates[0]}"
		CHOSEN_NOTE="preferred target unavailable for required storage"
		return 0
	fi

	for node in "${storage_candidates[@]}"; do
		if [[ "$node" == "$PREFERRED_TARGET" &&
			",${policy_nodes_csv}," == *",${node},"* ]]; then
			CHOSEN_DESTINATION="$node"
			return 0
		fi
	done

	for policy_node in "${policy_nodes[@]}"; do
		for node in "${storage_candidates[@]}"; do
			if [[ "$node" == "$policy_node" ]]; then
				CHOSEN_DESTINATION="$node"
				CHOSEN_NOTE="routed to satisfy HA node-affinity preference"
				return 0
			fi
		done
	done

	if (( ${#storage_candidates[@]} == 1 )); then
		CHOSEN_DESTINATION="${storage_candidates[0]}"
		CHOSEN_NOTE="only eligible node; HA node-affinity preference overridden"
		return 0
	fi

	if [[ "$policy_strict" == "1" ]]; then
		return 1
	fi

	for node in "${storage_candidates[@]}"; do
		if [[ "$node" == "$PREFERRED_TARGET" ]]; then
			CHOSEN_DESTINATION="$node"
			CHOSEN_NOTE="no preferred HA node was eligible; non-strict fallback used"
			return 0
		fi
	done
	CHOSEN_DESTINATION="${storage_candidates[0]}"
	CHOSEN_NOTE="no preferred HA node was eligible; non-strict fallback used"
}

get_node_available_memory() {
	local node="$1"

	pvesh get /nodes --output-format json 2>/dev/null |
		python3 -c '
import json
import sys

requested = sys.argv[1]
for node in json.load(sys.stdin):
    if str(node.get("node", "")) != requested:
        continue
    total = int(node.get("maxmem") or 0)
    used = int(node.get("mem") or 0)
    print(max(0, total - used))
    raise SystemExit
raise SystemExit(1)
' "$node"
}

format_bytes() {
	local bytes="${1:-0}"

	if command -v numfmt >/dev/null 2>&1; then
		numfmt --to=iec-i --suffix=B "$bytes"
	else
		printf '%d MiB' "$((bytes / 1024 / 1024))"
	fi
}

wait_for_ha_evacuation() {
	local deadline=$((SECONDS + HA_WAIT_SECONDS))
	local -a local_guests
	local -a ha_ids
	local -a remaining
	local guest
	local guest_status
	local guest_type
	local guest_vmid
	local ha_id

	log "Waiting for HA-managed guests to leave ${LOCAL_NODE}."

	while true; do
		load_lines local_guests get_local_guests || {
			warn "Could not refresh guests assigned to ${LOCAL_NODE}."
			return 1
		}
		load_lines ha_ids get_ha_guest_ids || {
			warn "Could not refresh HA resources."
			return 1
		}
		remaining=()

		for guest in "${local_guests[@]}"; do
			IFS=$'\x1f' read -r guest_vmid guest_type guest_status _ <<< "$guest"
			[[ "$guest_status" == "running" ]] || continue
			for ha_id in "${ha_ids[@]}"; do
				if [[ "$guest_vmid" == "$ha_id" ]]; then
					remaining+=("$guest")
					break
				fi
			done
		done

		(( ${#remaining[@]} == 0 )) && return

		if (( SECONDS >= deadline )); then
			warn "HA evacuation did not finish within ${HA_WAIT_SECONDS} seconds."
			printf 'HA-managed guests still assigned to %s:\n' "$LOCAL_NODE" >&2
			printf '  %s\n' "${remaining[@]}" >&2
			return 1
		fi

		printf '\r  Waiting: %d HA guest(s) remain... ' "${#remaining[@]}"
		sleep 5
	done
}

migrate_guest() {
	local vmid="$1"
	local guest_type="$2"
	local status="$3"
	local destination="$4"

	if [[ "$guest_type" == "qemu" ]]; then
		if [[ "$status" == "running" ]]; then
			qm migrate "$vmid" "$destination" --online
		else
			qm migrate "$vmid" "$destination"
		fi
	else
		if [[ "$status" == "running" ]]; then
			pct migrate "$vmid" "$destination" --restart 1 \
				--timeout "$SHUTDOWN_TIMEOUT"
		else
			pct migrate "$vmid" "$destination"
		fi
	fi
}

get_guest_node() {
	local vmid="$1"

	pvesh get /cluster/resources --type vm --output-format json 2>/dev/null |
		python3 -c '
import json
import sys

vmid = str(sys.argv[1])
for guest in json.load(sys.stdin):
    if str(guest.get("vmid", "")) == vmid:
        print(guest.get("node", ""))
        raise SystemExit
raise SystemExit(1)
' "$vmid"
}

wait_for_guest_node() {
	local vmid="$1"
	local destination="$2"
	local attempts="${3:-15}"
	local attempt=0

	while (( attempt < attempts )); do
		[[ "$(get_guest_node "$vmid")" == "$destination" ]] && return 0
		sleep 1
		((attempt++))
	done
	return 1
}

wait_for_migration_batch() {
	local index
	local pid
	local guest
	local log_file
	local vmid
	local guest_type
	local status
	local name
	local maxmem
	local required_storages
	local policy_nodes
	local policy_strict
	local policy_rule
	local destination
	local route_note

	for index in "${!batch_pids[@]}"; do
		pid="${batch_pids[$index]}"
		guest="${batch_guests[$index]}"
		log_file="${batch_logs[$index]}"
		IFS=$'\x1f' read -r vmid guest_type status name maxmem required_storages \
			policy_nodes policy_strict policy_rule destination route_note <<< "$guest"

		if wait "$pid" && wait_for_guest_node "$vmid" "$destination"; then
			log "Migration completed for ${guest_type} ${vmid} (${name:-unnamed}) to ${destination}."
			migration_successes+=("$guest")
		else
			warn "Migration failed for ${guest_type} ${vmid} (${name:-unnamed}) to ${destination}."
			migration_failures+=("$guest")
		fi

		if [[ -s "$log_file" ]]; then
			sed 's/^/  /' "$log_file"
		fi
	done

	batch_pids=()
	batch_guests=()
	batch_logs=()
}

shutdown_guest() {
	local vmid="$1"
	local guest_type="$2"

	if [[ "$guest_type" == "qemu" ]]; then
		qm shutdown "$vmid" --timeout "$SHUTDOWN_TIMEOUT"
	else
		pct shutdown "$vmid" --timeout "$SHUTDOWN_TIMEOUT"
	fi
}

guest_status() {
	local vmid="$1"
	local guest_type="$2"

	if [[ "$guest_type" == "qemu" ]]; then
		qm status "$vmid" 2>/dev/null | awk '{ print $2 }'
	else
		pct status "$vmid" 2>/dev/null | awk '{ print $2 }'
	fi
}

wait_for_guest_stopped() {
	local vmid="$1"
	local guest_type="$2"
	local attempts="${3:-15}"
	local attempt=0

	while (( attempt < attempts )); do
		[[ "$(guest_status "$vmid" "$guest_type")" == "stopped" ]] && return 0
		sleep 1
		((attempt++))
	done
	return 1
}

wait_for_maintenance_mode() {
	local deadline=$((SECONDS + MAINTENANCE_WAIT_SECONDS))

	log "Waiting for ${LOCAL_NODE} to enter HA maintenance mode."

	while true; do
		if node_in_maintenance "$LOCAL_NODE"; then
			printf '\n'
			return
		fi

		if (( SECONDS >= deadline )); then
			die "${LOCAL_NODE} did not enter HA maintenance mode within ${MAINTENANCE_WAIT_SECONDS} seconds."
		fi

		printf '\r  Waiting for HA maintenance mode... '
		sleep 2
	done
}

preflight() {
	local command

	for command in pvesh ha-manager python3 qm pct; do
		command -v "$command" >/dev/null 2>&1 ||
			die "${command} is required."
	done

	pvesh get /cluster/resources --type vm --output-format json >/dev/null 2>&1 ||
		die "Could not read cluster guest resources."
	pvesh get /cluster/ha/resources --output-format json >/dev/null 2>&1 ||
		die "Could not read HA resources."
	pvesh get /nodes --output-format json >/dev/null 2>&1 ||
		die "Could not read cluster node status."
	pvesh get /storage --output-format json >/dev/null 2>&1 ||
		die "Could not read cluster storage configuration."
	ha-manager status >/dev/null 2>&1 ||
		die "Could not read HA node status."
}

main() {
	local answer
	local available_memory
	local destination
	local guest
	local guest_type
	local log_file
	local maxmem
	local name
	local policy_nodes
	local policy_result
	local policy_rule
	local policy_strict
	local reason
	local required_memory
	local required_storages
	local route_note
	local scope
	local shared_csv
	local status
	local storage_result
	local vmid
	local -a guests=()
	local -a shared_storages=()
	local -a shared_guests=()
	local -a local_guests=()
	local -a unroutable_guests=()
	local -a migration_failures=()
	local -a migration_successes=()
	local -a migration_skipped=()
	local -a shutdown_failures=()
	local -a shutdown_successes=()
	local -a batch_pids=()
	local -a batch_guests=()
	local -a batch_logs=()
	local -a final_guests=()
	local -a running_final_guests=()
	local -a CURRENT_HA_IDS=()
	local -A DEST_REQUIRED_MEMORY=()

	MIGRATION_LOG_DIR="$(mktemp -d /tmp/ta-proxmenu-evacuation.XXXXXX)" ||
		die "Could not create the migration log directory."
	chmod 0700 "$MIGRATION_LOG_DIR"
	trap cleanup EXIT

	preflight
	wait_for_maintenance_mode
	wait_for_ha_evacuation ||
		die "Resolve the remaining HA guests before continuing the evacuation."

	load_lines shared_storages get_shared_storages ||
		die "Could not read shared-storage configuration."
	shared_csv="$(IFS=,; echo "${shared_storages[*]}")"
	load_lines guests get_local_guests ||
		die "Could not read guests assigned to ${LOCAL_NODE}."

	if (( ${#guests[@]} == 0 )); then
		log "No guests remain on ${LOCAL_NODE}."
		return 0
	fi

	for guest in "${guests[@]}"; do
		IFS=$'\x1f' read -r vmid guest_type status name maxmem <<< "$guest"
		storage_result="$(guest_storage_scope "$guest_type" "$vmid" "$shared_csv")" ||
			die "Could not inspect storage for ${guest_type} ${vmid}."
		IFS=$'\x1f' read -r scope reason required_storages <<< "$storage_result"

		if [[ "$scope" == "shared" ]]; then
			shared_guests+=(
				"${guest}"$'\x1f'"${required_storages}"
			)
		else
			local_guests+=(
				"${guest}"$'\x1f'"${reason:-local storage}"
			)
		fi
	done

	if (( ${#shared_guests[@]} > 0 )); then
		refresh_migration_nodes ||
			die "Could not build a safe migration-target list."
		select_target_node

		guests=("${shared_guests[@]}")
		shared_guests=()
		for guest in "${guests[@]}"; do
			IFS=$'\x1f' read -r vmid guest_type status name maxmem required_storages <<< "$guest"
			policy_result="$(get_guest_node_affinity "$guest_type" "$vmid")" ||
				die "Could not inspect HA node-affinity policy for ${guest_type} ${vmid}."
			IFS=$'\x1f' read -r policy_rule policy_strict policy_nodes <<< "$policy_result"

			if choose_destination "$required_storages" "$policy_nodes" "$policy_strict"; then
				destination="$CHOSEN_DESTINATION"
				route_note="$CHOSEN_NOTE"
				shared_guests+=(
					"${guest}"$'\x1f'"${policy_nodes}"$'\x1f'"${policy_strict}"$'\x1f'"${policy_rule}"$'\x1f'"${destination}"$'\x1f'"${route_note}"
				)
				if [[ "$status" == "running" && "$maxmem" =~ ^[0-9]+$ ]]; then
					DEST_REQUIRED_MEMORY["$destination"]="$(( ${DEST_REQUIRED_MEMORY[$destination]:-0} + maxmem ))"
				fi
			else
				unroutable_guests+=(
					"${guest}"$'\x1f'"${policy_nodes}"$'\x1f'"${policy_strict}"$'\x1f'"${policy_rule}"
				)
			fi
		done
	fi

	printf '\nEvacuation plan for %s:\n' "$LOCAL_NODE"
	[[ -n "$PREFERRED_TARGET" ]] &&
		printf '  Preferred migration target: %s\n' "$PREFERRED_TARGET"
	printf '  Shared-storage guests to migrate: %d\n' "${#shared_guests[@]}"
	printf '  Local-storage guests to shut down: %d\n' "${#local_guests[@]}"
	printf '  Shared guests without a valid route: %d\n' "${#unroutable_guests[@]}"

	if (( ${#shared_guests[@]} > 0 )); then
		printf '\nShared-storage guests:\n'
		for guest in "${shared_guests[@]}"; do
			IFS=$'\x1f' read -r vmid guest_type status name maxmem required_storages \
				policy_nodes policy_strict policy_rule destination route_note <<< "$guest"
			printf '  %-6s %-5s %-8s %-24s -> %s' \
				"$vmid" "$guest_type" "$status" "$name" "$destination"
			[[ -n "$route_note" ]] && printf '  (%s)' "$route_note"
			printf '\n'
		done
	fi

	if (( ${#local_guests[@]} > 0 )); then
		printf '\nLocal-storage guests (will not migrate):\n'
		for guest in "${local_guests[@]}"; do
			IFS=$'\x1f' read -r vmid guest_type status name maxmem reason <<< "$guest"
			printf '  %-6s %-5s %-8s %-24s %s\n' \
				"$vmid" "$guest_type" "$status" "$name" "$reason"
		done
	fi

	if (( ${#unroutable_guests[@]} > 0 )); then
		printf '\nShared guests with no eligible destination:\n'
		for guest in "${unroutable_guests[@]}"; do
			IFS=$'\x1f' read -r vmid guest_type status name maxmem required_storages \
				policy_nodes policy_strict policy_rule <<< "$guest"
			printf '  %-6s %-5s %-24s storages=%s' \
				"$vmid" "$guest_type" "$name" "$required_storages"
			[[ "$policy_rule" != "none" ]] &&
				printf ' policy=%s strict=%s nodes=%s' \
					"$policy_rule" "$policy_strict" "$policy_nodes"
			printf '\n'
		done
	fi

	for destination in "${!DEST_REQUIRED_MEMORY[@]}"; do
		required_memory="${DEST_REQUIRED_MEMORY[$destination]}"
		available_memory="$(get_node_available_memory "$destination" 2>/dev/null || echo 0)"
		printf '\nTarget %s memory: %s available; %s configured for incoming running guests.\n' \
			"$destination" "$(format_bytes "$available_memory")" \
			"$(format_bytes "$required_memory")"
		if (( available_memory > 0 && required_memory > available_memory )); then
			warn "${destination} has less currently available memory than the incoming guests' configured memory."
		fi
	done

	printf '\n'
	read -r -p "Proceed with guest migrations and local-storage guest shutdown? [y/N] " answer
	[[ "$answer" =~ ^[Yy]$ ]] || {
		echo "Evacuation cancelled; ${LOCAL_NODE} remains in maintenance mode."
		return 1
	}

	load_lines CURRENT_HA_IDS get_ha_guest_ids ||
		die "Could not safely recheck HA-managed guests."
	refresh_migration_nodes ||
		die "Could not safely recheck migration targets."

	for guest in "${shared_guests[@]}"; do
		IFS=$'\x1f' read -r vmid guest_type status name maxmem required_storages \
			policy_nodes policy_strict policy_rule destination route_note <<< "$guest"

		if guest_is_ha_managed "$vmid"; then
			status="$(guest_status "$vmid" "$guest_type")"
			if [[ "$status" != "stopped" ]]; then
				warn "${guest_type} ${vmid} is HA-managed and is not confirmed stopped; TAPM will not migrate it manually."
				migration_skipped+=("$guest")
				continue
			fi
			log "${guest_type} ${vmid} is HA-managed but stopped; continuing with offline migration."
		fi

		if ! choose_destination "$required_storages" "$policy_nodes" "$policy_strict"; then
			warn "No eligible destination remains for ${guest_type} ${vmid}; migration skipped."
			migration_skipped+=("$guest")
			continue
		fi
		if [[ "$destination" != "$CHOSEN_DESTINATION" ]]; then
			warn "Destination for ${guest_type} ${vmid} changed from ${destination} to ${CHOSEN_DESTINATION} after recheck."
			destination="$CHOSEN_DESTINATION"
			route_note="$CHOSEN_NOTE"
			guest="${vmid}"$'\x1f'"${guest_type}"$'\x1f'"${status}"$'\x1f'"${name}"$'\x1f'"${maxmem}"$'\x1f'"${required_storages}"$'\x1f'"${policy_nodes}"$'\x1f'"${policy_strict}"$'\x1f'"${policy_rule}"$'\x1f'"${destination}"$'\x1f'"${route_note}"
		fi

		log "Starting migration for ${guest_type} ${vmid} (${name:-unnamed}) to ${destination}."
		log_file="${MIGRATION_LOG_DIR}/${guest_type}-${vmid}.log"
		migrate_guest "$vmid" "$guest_type" "$status" "$destination" >"$log_file" 2>&1 &
		batch_pids+=("$!")
		batch_guests+=("$guest")
		batch_logs+=("$log_file")

		if (( ${#batch_pids[@]} >= MAX_PARALLEL_MIGRATIONS )); then
			wait_for_migration_batch
		fi
	done
	(( ${#batch_pids[@]} == 0 )) || wait_for_migration_batch

	for guest in "${local_guests[@]}"; do
		IFS=$'\x1f' read -r vmid guest_type status name maxmem reason <<< "$guest"
		status="$(guest_status "$vmid" "$guest_type")"
		[[ "$status" == "running" ]] || continue

		log "Shutting down local-storage ${guest_type} ${vmid} (${name:-unnamed})."
		if shutdown_guest "$vmid" "$guest_type" &&
			wait_for_guest_stopped "$vmid" "$guest_type"; then
			shutdown_successes+=("$guest")
		else
			warn "Graceful shutdown failed for ${guest_type} ${vmid}; it was not force-stopped."
			shutdown_failures+=("$guest")
		fi
	done

	load_lines final_guests get_local_guests ||
		die "Could not verify the final guest state on ${LOCAL_NODE}."
	for guest in "${final_guests[@]}"; do
		IFS=$'\x1f' read -r vmid guest_type status name maxmem <<< "$guest"
		[[ "$status" == "running" ]] && running_final_guests+=("$guest")
	done

	printf '\nEvacuation summary:\n'
	printf '  Successful migrations:       %d\n' "${#migration_successes[@]}"
	printf '  Failed migrations:           %d\n' "${#migration_failures[@]}"
	printf '  Skipped migrations:          %d\n' "${#migration_skipped[@]}"
	printf '  Unroutable shared guests:    %d\n' "${#unroutable_guests[@]}"
	printf '  Local guest shutdowns:       %d\n' "${#shutdown_successes[@]}"
	printf '  Failed local shutdowns:      %d\n' "${#shutdown_failures[@]}"
	printf '  Guests still running locally: %d\n' "${#running_final_guests[@]}"

	if (( ${#migration_successes[@]} > 0 )); then
		printf '\nMigrated guests:\n'
		for guest in "${migration_successes[@]}"; do
			IFS=$'\x1f' read -r vmid guest_type status name maxmem required_storages \
				policy_nodes policy_strict policy_rule destination route_note <<< "$guest"
			printf '  %-6s %-5s %-24s -> %s\n' \
				"$vmid" "$guest_type" "$name" "$destination"
		done
	fi

	if (( ${#migration_failures[@]} > 0 )); then
		printf '\nFailed migrations (left unchanged and not force-stopped):\n'
		for guest in "${migration_failures[@]}"; do
			IFS=$'\x1f' read -r vmid guest_type status name maxmem required_storages \
				policy_nodes policy_strict policy_rule destination route_note <<< "$guest"
			printf '  %-6s %-5s %-24s target=%s\n' \
				"$vmid" "$guest_type" "$name" "$destination"
		done
	fi

	if (( ${#shutdown_successes[@]} > 0 )); then
		printf '\nLocal-storage guests shut down:\n'
		for guest in "${shutdown_successes[@]}"; do
			IFS=$'\x1f' read -r vmid guest_type status name maxmem reason <<< "$guest"
			printf '  %-6s %-5s %s\n' "$vmid" "$guest_type" "$name"
		done
	fi

	if (( ${#final_guests[@]} > 0 )); then
		printf '\nGuests still assigned to %s:\n' "$LOCAL_NODE"
		for guest in "${final_guests[@]}"; do
			IFS=$'\x1f' read -r vmid guest_type status name maxmem <<< "$guest"
			printf '  %-6s %-5s %-8s %s\n' "$vmid" "$guest_type" "$status" "$name"
		done
	fi

	printf '\nNo guest was force-stopped.\n'

	if (( ${#migration_failures[@]} > 0 ||
		${#migration_skipped[@]} > 0 ||
		${#unroutable_guests[@]} > 0 ||
		${#shutdown_failures[@]} > 0 ||
		${#running_final_guests[@]} > 0 )); then
		return 1
	fi
}

if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
	main "$@"
fi