TAI/TA-ProxMenu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update

Browse Source
This commit is contained in:
David Schroeder
2026-07-25 17:48:04 -05:00
parent bd005495ce
commit d19400a732
5 changed files with 1314 additions and 250 deletions
Download Patch File Download Diff File Expand all files Collapse all files
inc/evacuate-proxmox-node.sh
Executable → Regular
+669 -153
View File
@@ -1,18 +1,21 @@
#!/usr/bin/env bash
set -u
set -u -o pipefail
# Evacuate non-HA guests after the local Proxmox node enters HA maintenance.
# Guests using shared storage are migrated to one operator-selected node.
# Guests with local storage remain on this node and are gracefully shut down.
# HA-managed guests remain under Proxmox HA control. Non-HA guests using
# shared storage are migrated, while guests using local storage are shut down.
HA_WAIT_SECONDS=300
MAINTENANCE_WAIT_SECONDS=60
MAX_PARALLEL_MIGRATIONS=3
SHUTDOWN_TIMEOUT=180
LOCAL_NODE="$(hostname -s)"
MIGRATION_LOG_DIR="$(mktemp -d)"
MIGRATION_LOG_DIR=''
PREFERRED_TARGET=''
HA_RULES_FILE="${HA_RULES_FILE:-/etc/pve/ha/rules.cfg}"
trap 'rm -rf "$MIGRATION_LOG_DIR"' EXIT
declare -a MIGRATION_NODES=()
declare -A NODE_STORAGE_CACHE=()
log() {
printf '\n[%s] %s\n' "$(date '+%F %T')" "$*"
@@ -27,18 +30,24 @@ die() {
exit 1
}
command -v pvesh >/dev/null 2>&1 || die "pvesh is required."
command -v ha-manager >/dev/null 2>&1 || die "ha-manager is required."
command -v python3 >/dev/null 2>&1 || die "python3 is required."
load_lines() {
local destination_name="$1"
local output
local -n destination_ref="$destination_name"
shift
pvesh get /cluster/resources --type vm --output-format json >/dev/null 2>&1 ||
die "Could not read cluster guest resources."
pvesh get /cluster/ha/resources --output-format json >/dev/null 2>&1 ||
die "Could not read HA resources."
pvesh get /nodes --output-format json >/dev/null 2>&1 ||
die "Could not read cluster node status."
pvesh get /storage --output-format json >/dev/null 2>&1 ||
die "Could not read cluster storage configuration."
output="$("$@")" || return 1
destination_ref=()
# shellcheck disable=SC2034 # mapfile writes through the nameref.
[[ -z "$output" ]] || mapfile -t destination_ref <<< "$output"
}
cleanup() {
if [[ "$MIGRATION_LOG_DIR" == /tmp/ta-proxmenu-evacuation.* &&
-d "$MIGRATION_LOG_DIR" ]]; then
rm -rf -- "$MIGRATION_LOG_DIR"
fi
}
get_local_guests() {
pvesh get /cluster/resources --type vm --output-format json 2>/dev/null |
@@ -55,6 +64,7 @@ for guest in json.load(sys.stdin):
guest.get("type", ""),
guest.get("status", "unknown"),
str(guest.get("name", "")).replace("\x1f", " "),
int(guest.get("maxmem") or 0),
sep="\x1f",
)
' "$LOCAL_NODE"
@@ -75,6 +85,16 @@ for resource in json.load(sys.stdin):
'
}
guest_is_ha_managed() {
local vmid="$1"
local ha_id
for ha_id in "${CURRENT_HA_IDS[@]:-}"; do
[[ "$ha_id" == "$vmid" ]] && return 0
done
return 1
}
get_online_nodes() {
pvesh get /nodes --output-format json 2>/dev/null |
python3 -c '
@@ -89,6 +109,14 @@ for node in json.load(sys.stdin):
' "$LOCAL_NODE"
}
node_in_maintenance() {
local node="$1"
ha-manager status 2>/dev/null |
grep -F "lrm ${node} " |
grep -q "maintenance mode"
}
get_shared_storages() {
pvesh get /storage --output-format json 2>/dev/null |
python3 -c '
@@ -101,6 +129,74 @@ for storage in json.load(sys.stdin):
'
}
get_node_active_storages() {
local node="$1"
pvesh get "/nodes/${node}/storage" --output-format json 2>/dev/null |
python3 -c '
import json
import sys
for storage in json.load(sys.stdin):
active = storage.get("active")
enabled = storage.get("enabled", 1)
if active in (1, True, "1") and enabled not in (0, False, "0"):
storage_id = str(storage.get("storage", ""))
if storage_id:
print(storage_id)
'
}
refresh_migration_nodes() {
local ha_status
local node
local online_output
local storage_csv
local -a online_nodes=()
local -a usable_nodes=()
online_output="$(get_online_nodes)" || {
warn "Could not refresh online cluster nodes."
return 1
}
[[ -z "$online_output" ]] ||
mapfile -t online_nodes <<< "$online_output"
ha_status="$(ha-manager status 2>/dev/null)" || {
warn "Could not refresh HA node status."
return 1
}
NODE_STORAGE_CACHE=()
for node in "${online_nodes[@]}"; do
if grep -F "lrm ${node} " <<< "$ha_status" |
grep -q "maintenance mode"; then
continue
fi
storage_csv="$(get_node_active_storages "$node" | paste -sd, -)" || {
warn "Could not read storage status from ${node}; it will not be used."
continue
}
NODE_STORAGE_CACHE["$node"]="$storage_csv"
usable_nodes+=("$node")
done
MIGRATION_NODES=("${usable_nodes[@]}")
}
node_has_required_storages() {
local node="$1"
local required_csv="$2"
local available_csv="${NODE_STORAGE_CACHE[$node]-}"
local storage
local -a required_storages=()
[[ -n "$required_csv" ]] || return 0
IFS=',' read -r -a required_storages <<< "$required_csv"
for storage in "${required_storages[@]}"; do
[[ ",${available_csv}," == *",${storage},"* ]] || return 1
done
}
guest_storage_scope() {
local guest_type="$1"
local vmid="$2"
@@ -125,6 +221,7 @@ else:
disk_key = re.compile(r"^(?:rootfs|mp\d+|unused\d+)$")
local_reasons = []
required_storages = set()
for key, raw_value in config.items():
if not disk_key.match(key) or not isinstance(raw_value, str):
continue
@@ -137,41 +234,221 @@ for key, raw_value in config.items():
continue
storage = volume.split(":", 1)[0]
required_storages.add(storage)
if storage not in shared:
local_reasons.append(f"{key}={storage}")
if local_reasons:
print("local\x1f" + ", ".join(local_reasons))
else:
print("shared\x1f")
scope = "local" if local_reasons else "shared"
print(
scope,
", ".join(local_reasons),
",".join(sorted(required_storages)),
sep="\x1f",
)
' "$guest_type" "$shared_csv"
}
get_guest_node_affinity() {
local guest_type="$1"
local vmid="$2"
local sid_type='vm'
[[ "$guest_type" == "lxc" ]] && sid_type='ct'
python3 -c '
import os
import sys
sid = sys.argv[1]
path = sys.argv[2]
if not os.path.exists(path):
print("none", "0", "", sep="\x1f")
raise SystemExit
rules = []
current = None
with open(path, encoding="utf-8") as handle:
for raw_line in handle:
line = raw_line.rstrip()
if not line or line.lstrip().startswith("#"):
continue
if not line[0].isspace() and ":" in line:
rule_type, rule_id = line.split(":", 1)
current = {
"type": rule_type.strip(),
"id": rule_id.strip(),
}
rules.append(current)
continue
if current is not None and line[0].isspace():
key_value = line.strip().split(None, 1)
if len(key_value) == 2:
current[key_value[0]] = key_value[1].strip()
for rule in rules:
if rule.get("type") != "node-affinity":
continue
if rule.get("disable", "0") in ("1", "yes", "true", "on"):
continue
resources = {
item.strip()
for item in rule.get("resources", "").replace(";", ",").split(",")
if item.strip()
}
if sid not in resources:
continue
nodes = []
for position, item in enumerate(rule.get("nodes", "").split(",")):
item = item.strip()
if not item:
continue
node = item
priority = 0
if ":" in item:
possible_node, possible_priority = item.rsplit(":", 1)
try:
priority = int(possible_priority)
node = possible_node
except ValueError:
pass
nodes.append((node, priority, position))
nodes.sort(key=lambda value: (-value[1], value[2]))
strict = "1" if rule.get("strict", "0") in ("1", "yes", "true", "on") else "0"
print(rule.get("id", "node-affinity"), strict, ",".join(n[0] for n in nodes), sep="\x1f")
raise SystemExit
print("none", "0", "", sep="\x1f")
' "${sid_type}:${vmid}" "$HA_RULES_FILE"
}
select_target_node() {
local -a nodes
local choice
local index
mapfile -t nodes < <(get_online_nodes)
(( ${#nodes[@]} > 0 )) || die "No other online cluster node is available."
(( ${#MIGRATION_NODES[@]} > 0 )) ||
die "No other online, non-maintenance cluster node is available."
printf '\nAvailable migration targets:\n\n'
for index in "${!nodes[@]}"; do
printf ' %d) %s\n' "$((index + 1))" "${nodes[$index]}"
printf '\nAvailable preferred migration targets:\n\n'
for index in "${!MIGRATION_NODES[@]}"; do
printf ' %d) %s\n' "$((index + 1))" "${MIGRATION_NODES[$index]}"
done
while true; do
printf '\n'
read -r -p "Select migration target [1-${#nodes[@]}]: " choice
read -r -p "Select preferred migration target [1-${#MIGRATION_NODES[@]}]: " choice
if [[ "$choice" =~ ^[0-9]+$ ]] &&
(( choice >= 1 && choice <= ${#nodes[@]} )); then
TARGET_NODE="${nodes[$((choice - 1))]}"
(( choice >= 1 && choice <= ${#MIGRATION_NODES[@]} )); then
PREFERRED_TARGET="${MIGRATION_NODES[$((choice - 1))]}"
return
fi
printf 'Invalid selection.\n' >&2
done
}
CHOSEN_DESTINATION=''
CHOSEN_NOTE=''
choose_destination() {
local required_csv="$1"
local policy_nodes_csv="$2"
local policy_strict="$3"
local node
local policy_node
local -a storage_candidates=()
local -a policy_nodes=()
CHOSEN_DESTINATION=''
CHOSEN_NOTE=''
for node in "${MIGRATION_NODES[@]}"; do
if node_has_required_storages "$node" "$required_csv"; then
storage_candidates+=("$node")
fi
done
(( ${#storage_candidates[@]} > 0 )) || return 1
IFS=',' read -r -a policy_nodes <<< "$policy_nodes_csv"
if [[ -z "$policy_nodes_csv" ]]; then
for node in "${storage_candidates[@]}"; do
if [[ "$node" == "$PREFERRED_TARGET" ]]; then
CHOSEN_DESTINATION="$node"
return 0
fi
done
CHOSEN_DESTINATION="${storage_candidates[0]}"
CHOSEN_NOTE="preferred target unavailable for required storage"
return 0
fi
for node in "${storage_candidates[@]}"; do
if [[ "$node" == "$PREFERRED_TARGET" &&
",${policy_nodes_csv}," == *",${node},"* ]]; then
CHOSEN_DESTINATION="$node"
return 0
fi
done
for policy_node in "${policy_nodes[@]}"; do
for node in "${storage_candidates[@]}"; do
if [[ "$node" == "$policy_node" ]]; then
CHOSEN_DESTINATION="$node"
CHOSEN_NOTE="routed to satisfy HA node-affinity preference"
return 0
fi
done
done
if (( ${#storage_candidates[@]} == 1 )); then
CHOSEN_DESTINATION="${storage_candidates[0]}"
CHOSEN_NOTE="only eligible node; HA node-affinity preference overridden"
return 0
fi
if [[ "$policy_strict" == "1" ]]; then
return 1
fi
for node in "${storage_candidates[@]}"; do
if [[ "$node" == "$PREFERRED_TARGET" ]]; then
CHOSEN_DESTINATION="$node"
CHOSEN_NOTE="no preferred HA node was eligible; non-strict fallback used"
return 0
fi
done
CHOSEN_DESTINATION="${storage_candidates[0]}"
CHOSEN_NOTE="no preferred HA node was eligible; non-strict fallback used"
}
get_node_available_memory() {
local node="$1"
pvesh get /nodes --output-format json 2>/dev/null |
python3 -c '
import json
import sys
requested = sys.argv[1]
for node in json.load(sys.stdin):
if str(node.get("node", "")) != requested:
continue
total = int(node.get("maxmem") or 0)
used = int(node.get("mem") or 0)
print(max(0, total - used))
raise SystemExit
raise SystemExit(1)
' "$node"
}
format_bytes() {
local bytes="${1:-0}"
if command -v numfmt >/dev/null 2>&1; then
numfmt --to=iec-i --suffix=B "$bytes"
else
printf '%d MiB' "$((bytes / 1024 / 1024))"
fi
}
wait_for_ha_evacuation() {
local deadline=$((SECONDS + HA_WAIT_SECONDS))
local -a local_guests
@@ -183,8 +460,14 @@ wait_for_ha_evacuation() {
log "Waiting for HA-managed guests to leave ${LOCAL_NODE}."
while true; do
mapfile -t local_guests < <(get_local_guests)
mapfile -t ha_ids < <(get_ha_guest_ids)
load_lines local_guests get_local_guests || {
warn "Could not refresh guests assigned to ${LOCAL_NODE}."
return 1
}
load_lines ha_ids get_ha_guest_ids || {
warn "Could not refresh HA resources."
return 1
}
remaining=()
for guest in "${local_guests[@]}"; do
@@ -214,23 +497,55 @@ migrate_guest() {
local vmid="$1"
local guest_type="$2"
local status="$3"
local destination="$4"
if [[ "$guest_type" == "qemu" ]]; then
if [[ "$status" == "running" ]]; then
qm migrate "$vmid" "$TARGET_NODE" --online
qm migrate "$vmid" "$destination" --online
else
qm migrate "$vmid" "$TARGET_NODE"
qm migrate "$vmid" "$destination"
fi
else
if [[ "$status" == "running" ]]; then
pct migrate "$vmid" "$TARGET_NODE" --restart 1 \
pct migrate "$vmid" "$destination" --restart 1 \
--timeout "$SHUTDOWN_TIMEOUT"
else
pct migrate "$vmid" "$TARGET_NODE"
pct migrate "$vmid" "$destination"
fi
fi
}
get_guest_node() {
local vmid="$1"
pvesh get /cluster/resources --type vm --output-format json 2>/dev/null |
python3 -c '
import json
import sys
vmid = str(sys.argv[1])
for guest in json.load(sys.stdin):
if str(guest.get("vmid", "")) == vmid:
print(guest.get("node", ""))
raise SystemExit
raise SystemExit(1)
' "$vmid"
}
wait_for_guest_node() {
local vmid="$1"
local destination="$2"
local attempts="${3:-15}"
local attempt=0
while (( attempt < attempts )); do
[[ "$(get_guest_node "$vmid")" == "$destination" ]] && return 0
sleep 1
((attempt++))
done
return 1
}
wait_for_migration_batch() {
local index
local pid
@@ -240,17 +555,26 @@ wait_for_migration_batch() {
local guest_type
local status
local name
local maxmem
local required_storages
local policy_nodes
local policy_strict
local policy_rule
local destination
local route_note
for index in "${!batch_pids[@]}"; do
pid="${batch_pids[$index]}"
guest="${batch_guests[$index]}"
log_file="${batch_logs[$index]}"
IFS=$'\x1f' read -r vmid guest_type status name <<< "$guest"
IFS=$'\x1f' read -r vmid guest_type status name maxmem required_storages \
policy_nodes policy_strict policy_rule destination route_note <<< "$guest"
if wait "$pid"; then
log "Migration completed for ${guest_type} ${vmid} (${name:-unnamed})."
if wait "$pid" && wait_for_guest_node "$vmid" "$destination"; then
log "Migration completed for ${guest_type} ${vmid} (${name:-unnamed}) to ${destination}."
migration_successes+=("$guest")
else
warn "Migration failed for ${guest_type} ${vmid} (${name:-unnamed})."
warn "Migration failed for ${guest_type} ${vmid} (${name:-unnamed}) to ${destination}."
migration_failures+=("$guest")
fi
@@ -286,15 +610,27 @@ guest_status() {
fi
}
wait_for_guest_stopped() {
local vmid="$1"
local guest_type="$2"
local attempts="${3:-15}"
local attempt=0
while (( attempt < attempts )); do
[[ "$(guest_status "$vmid" "$guest_type")" == "stopped" ]] && return 0
sleep 1
((attempt++))
done
return 1
}
wait_for_maintenance_mode() {
local deadline=$((SECONDS + MAINTENANCE_WAIT_SECONDS))
log "Waiting for ${LOCAL_NODE} to enter HA maintenance mode."
while true; do
if ha-manager status |
grep -F "$LOCAL_NODE" |
grep -q "maintenance mode"; then
if node_in_maintenance "$LOCAL_NODE"; then
printf '\n'
return
fi
@@ -308,132 +644,312 @@ wait_for_maintenance_mode() {
done
}
wait_for_maintenance_mode
preflight() {
local command
wait_for_ha_evacuation ||
die "Resolve the remaining HA guests before continuing the evacuation."
select_target_node
mapfile -t shared_storages < <(get_shared_storages)
shared_csv="$(IFS=,; echo "${shared_storages[*]}")"
mapfile -t guests < <(get_local_guests)
if (( ${#guests[@]} == 0 )); then
log "No guests remain on ${LOCAL_NODE}."
exit 0
fi
declare -a shared_guests=()
declare -a local_guests=()
for guest in "${guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name <<< "$guest"
storage_result="$(guest_storage_scope "$guest_type" "$vmid" "$shared_csv")" ||
die "Could not inspect storage for ${guest_type} ${vmid}."
IFS=$'\x1f' read -r scope reason <<< "$storage_result"
if [[ "$scope" == "shared" ]]; then
shared_guests+=("$guest")
else
local_guests+=("${guest}"$'\x1f'"${reason:-local storage}")
fi
done
printf '\nEvacuation plan for %s:\n' "$LOCAL_NODE"
printf ' Migration target: %s\n' "$TARGET_NODE"
printf ' Shared-storage guests to migrate: %d\n' "${#shared_guests[@]}"
printf ' Local-storage guests to retain: %d\n' "${#local_guests[@]}"
if (( ${#shared_guests[@]} > 0 )); then
printf '\nShared-storage guests:\n'
for guest in "${shared_guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name <<< "$guest"
printf ' %-6s %-5s %-8s %s\n' "$vmid" "$guest_type" "$status" "$name"
for command in pvesh ha-manager python3 qm pct; do
command -v "$command" >/dev/null 2>&1 ||
die "${command} is required."
done
fi
if (( ${#local_guests[@]} > 0 )); then
printf '\nLocal-storage guests (will not migrate):\n'
for guest in "${local_guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name reason <<< "$guest"
printf ' %-6s %-5s %-8s %-24s %s\n' \
"$vmid" "$guest_type" "$status" "$name" "$reason"
pvesh get /cluster/resources --type vm --output-format json >/dev/null 2>&1 ||
die "Could not read cluster guest resources."
pvesh get /cluster/ha/resources --output-format json >/dev/null 2>&1 ||
die "Could not read HA resources."
pvesh get /nodes --output-format json >/dev/null 2>&1 ||
die "Could not read cluster node status."
pvesh get /storage --output-format json >/dev/null 2>&1 ||
die "Could not read cluster storage configuration."
ha-manager status >/dev/null 2>&1 ||
die "Could not read HA node status."
}
main() {
local answer
local available_memory
local destination
local guest
local guest_type
local log_file
local maxmem
local name
local policy_nodes
local policy_result
local policy_rule
local policy_strict
local reason
local required_memory
local required_storages
local route_note
local scope
local shared_csv
local status
local storage_result
local vmid
local -a guests=()
local -a shared_storages=()
local -a shared_guests=()
local -a local_guests=()
local -a unroutable_guests=()
local -a migration_failures=()
local -a migration_successes=()
local -a migration_skipped=()
local -a shutdown_failures=()
local -a shutdown_successes=()
local -a batch_pids=()
local -a batch_guests=()
local -a batch_logs=()
local -a final_guests=()
local -a running_final_guests=()
local -a CURRENT_HA_IDS=()
local -A DEST_REQUIRED_MEMORY=()
MIGRATION_LOG_DIR="$(mktemp -d /tmp/ta-proxmenu-evacuation.XXXXXX)" ||
die "Could not create the migration log directory."
chmod 0700 "$MIGRATION_LOG_DIR"
trap cleanup EXIT
preflight
wait_for_maintenance_mode
wait_for_ha_evacuation ||
die "Resolve the remaining HA guests before continuing the evacuation."
load_lines shared_storages get_shared_storages ||
die "Could not read shared-storage configuration."
shared_csv="$(IFS=,; echo "${shared_storages[*]}")"
load_lines guests get_local_guests ||
die "Could not read guests assigned to ${LOCAL_NODE}."
if (( ${#guests[@]} == 0 )); then
log "No guests remain on ${LOCAL_NODE}."
return 0
fi
for guest in "${guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name maxmem <<< "$guest"
storage_result="$(guest_storage_scope "$guest_type" "$vmid" "$shared_csv")" ||
die "Could not inspect storage for ${guest_type} ${vmid}."
IFS=$'\x1f' read -r scope reason required_storages <<< "$storage_result"
if [[ "$scope" == "shared" ]]; then
shared_guests+=(
"${guest}"$'\x1f'"${required_storages}"
)
else
local_guests+=(
"${guest}"$'\x1f'"${reason:-local storage}"
)
fi
done
fi
printf '\n'
read -r -p "Proceed with shared-storage guest migration? [y/N] " answer
[[ "$answer" =~ ^[Yy]$ ]] || { echo "Evacuation cancelled."; exit 0; }
if (( ${#shared_guests[@]} > 0 )); then
refresh_migration_nodes ||
die "Could not build a safe migration-target list."
select_target_node
declare -a migration_failures=()
declare -a batch_pids=()
declare -a batch_guests=()
declare -a batch_logs=()
guests=("${shared_guests[@]}")
shared_guests=()
for guest in "${guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name maxmem required_storages <<< "$guest"
policy_result="$(get_guest_node_affinity "$guest_type" "$vmid")" ||
die "Could not inspect HA node-affinity policy for ${guest_type} ${vmid}."
IFS=$'\x1f' read -r policy_rule policy_strict policy_nodes <<< "$policy_result"
for guest in "${shared_guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name <<< "$guest"
log "Starting migration for ${guest_type} ${vmid} (${name:-unnamed}) to ${TARGET_NODE}."
log_file="${MIGRATION_LOG_DIR}/${guest_type}-${vmid}.log"
migrate_guest "$vmid" "$guest_type" "$status" >"$log_file" 2>&1 &
batch_pids+=("$!")
batch_guests+=("$guest")
batch_logs+=("$log_file")
if (( ${#batch_pids[@]} >= MAX_PARALLEL_MIGRATIONS )); then
wait_for_migration_batch
if choose_destination "$required_storages" "$policy_nodes" "$policy_strict"; then
destination="$CHOSEN_DESTINATION"
route_note="$CHOSEN_NOTE"
shared_guests+=(
"${guest}"$'\x1f'"${policy_nodes}"$'\x1f'"${policy_strict}"$'\x1f'"${policy_rule}"$'\x1f'"${destination}"$'\x1f'"${route_note}"
)
if [[ "$status" == "running" && "$maxmem" =~ ^[0-9]+$ ]]; then
DEST_REQUIRED_MEMORY["$destination"]="$(( ${DEST_REQUIRED_MEMORY[$destination]:-0} + maxmem ))"
fi
else
unroutable_guests+=(
"${guest}"$'\x1f'"${policy_nodes}"$'\x1f'"${policy_strict}"$'\x1f'"${policy_rule}"
)
fi
done
fi
done
(( ${#batch_pids[@]} == 0 )) || wait_for_migration_batch
printf '\nEvacuation plan for %s:\n' "$LOCAL_NODE"
[[ -n "$PREFERRED_TARGET" ]] &&
printf ' Preferred migration target: %s\n' "$PREFERRED_TARGET"
printf ' Shared-storage guests to migrate: %d\n' "${#shared_guests[@]}"
printf ' Local-storage guests to shut down: %d\n' "${#local_guests[@]}"
printf ' Shared guests without a valid route: %d\n' "${#unroutable_guests[@]}"
declare -a running_local_guests=()
for guest in "${local_guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name reason <<< "$guest"
status="$(guest_status "$vmid" "$guest_type")"
if [[ "$status" == "running" ]]; then
running_local_guests+=(
"${vmid}"$'\x1f'"${guest_type}"$'\x1f'"${status}"$'\x1f'"${name}"$'\x1f'"${reason}"
)
if (( ${#shared_guests[@]} > 0 )); then
printf '\nShared-storage guests:\n'
for guest in "${shared_guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name maxmem required_storages \
policy_nodes policy_strict policy_rule destination route_note <<< "$guest"
printf ' %-6s %-5s %-8s %-24s -> %s' \
"$vmid" "$guest_type" "$status" "$name" "$destination"
[[ -n "$route_note" ]] && printf ' (%s)' "$route_note"
printf '\n'
done
fi
done
if (( ${#running_local_guests[@]} > 0 )); then
printf '\nThe following local-storage guests remain running:\n'
for guest in "${running_local_guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name reason <<< "$guest"
printf ' %-6s %-5s %-24s %s\n' "$vmid" "$guest_type" "$name" "$reason"
if (( ${#local_guests[@]} > 0 )); then
printf '\nLocal-storage guests (will not migrate):\n'
for guest in "${local_guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name maxmem reason <<< "$guest"
printf ' %-6s %-5s %-8s %-24s %s\n' \
"$vmid" "$guest_type" "$status" "$name" "$reason"
done
fi
if (( ${#unroutable_guests[@]} > 0 )); then
printf '\nShared guests with no eligible destination:\n'
for guest in "${unroutable_guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name maxmem required_storages \
policy_nodes policy_strict policy_rule <<< "$guest"
printf ' %-6s %-5s %-24s storages=%s' \
"$vmid" "$guest_type" "$name" "$required_storages"
[[ "$policy_rule" != "none" ]] &&
printf ' policy=%s strict=%s nodes=%s' \
"$policy_rule" "$policy_strict" "$policy_nodes"
printf '\n'
done
fi
for destination in "${!DEST_REQUIRED_MEMORY[@]}"; do
required_memory="${DEST_REQUIRED_MEMORY[$destination]}"
available_memory="$(get_node_available_memory "$destination" 2>/dev/null || echo 0)"
printf '\nTarget %s memory: %s available; %s configured for incoming running guests.\n' \
"$destination" "$(format_bytes "$available_memory")" \
"$(format_bytes "$required_memory")"
if (( available_memory > 0 && required_memory > available_memory )); then
warn "${destination} has less currently available memory than the incoming guests' configured memory."
fi
done
printf '\n'
read -r -p "Gracefully shut down these local-storage guests? [y/N] " answer
if [[ "$answer" =~ ^[Yy]$ ]]; then
for guest in "${running_local_guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name reason <<< "$guest"
log "Shutting down ${guest_type} ${vmid} (${name:-unnamed})."
shutdown_guest "$vmid" "$guest_type" ||
warn "Graceful shutdown failed for ${guest_type} ${vmid}; it was not force-stopped."
read -r -p "Proceed with guest migrations and local-storage guest shutdown? [y/N] " answer
[[ "$answer" =~ ^[Yy]$ ]] || {
echo "Evacuation cancelled; ${LOCAL_NODE} remains in maintenance mode."
return 1
}
load_lines CURRENT_HA_IDS get_ha_guest_ids ||
die "Could not safely recheck HA-managed guests."
refresh_migration_nodes ||
die "Could not safely recheck migration targets."
for guest in "${shared_guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name maxmem required_storages \
policy_nodes policy_strict policy_rule destination route_note <<< "$guest"
if guest_is_ha_managed "$vmid"; then
warn "${guest_type} ${vmid} became HA-managed; TAPM will not migrate it manually."
migration_skipped+=("$guest")
continue
fi
if ! choose_destination "$required_storages" "$policy_nodes" "$policy_strict"; then
warn "No eligible destination remains for ${guest_type} ${vmid}; migration skipped."
migration_skipped+=("$guest")
continue
fi
if [[ "$destination" != "$CHOSEN_DESTINATION" ]]; then
warn "Destination for ${guest_type} ${vmid} changed from ${destination} to ${CHOSEN_DESTINATION} after recheck."
destination="$CHOSEN_DESTINATION"
route_note="$CHOSEN_NOTE"
guest="${vmid}"$'\x1f'"${guest_type}"$'\x1f'"${status}"$'\x1f'"${name}"$'\x1f'"${maxmem}"$'\x1f'"${required_storages}"$'\x1f'"${policy_nodes}"$'\x1f'"${policy_strict}"$'\x1f'"${policy_rule}"$'\x1f'"${destination}"$'\x1f'"${route_note}"
fi
log "Starting migration for ${guest_type} ${vmid} (${name:-unnamed}) to ${destination}."
log_file="${MIGRATION_LOG_DIR}/${guest_type}-${vmid}.log"
migrate_guest "$vmid" "$guest_type" "$status" "$destination" >"$log_file" 2>&1 &
batch_pids+=("$!")
batch_guests+=("$guest")
batch_logs+=("$log_file")
if (( ${#batch_pids[@]} >= MAX_PARALLEL_MIGRATIONS )); then
wait_for_migration_batch
fi
done
(( ${#batch_pids[@]} == 0 )) || wait_for_migration_batch
for guest in "${local_guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name maxmem reason <<< "$guest"
status="$(guest_status "$vmid" "$guest_type")"
[[ "$status" == "running" ]] || continue
log "Shutting down local-storage ${guest_type} ${vmid} (${name:-unnamed})."
if shutdown_guest "$vmid" "$guest_type" &&
wait_for_guest_stopped "$vmid" "$guest_type"; then
shutdown_successes+=("$guest")
else
warn "Graceful shutdown failed for ${guest_type} ${vmid}; it was not force-stopped."
shutdown_failures+=("$guest")
fi
done
load_lines final_guests get_local_guests ||
die "Could not verify the final guest state on ${LOCAL_NODE}."
for guest in "${final_guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name maxmem <<< "$guest"
[[ "$status" == "running" ]] && running_final_guests+=("$guest")
done
printf '\nEvacuation summary:\n'
printf ' Successful migrations: %d\n' "${#migration_successes[@]}"
printf ' Failed migrations: %d\n' "${#migration_failures[@]}"
printf ' Skipped migrations: %d\n' "${#migration_skipped[@]}"
printf ' Unroutable shared guests: %d\n' "${#unroutable_guests[@]}"
printf ' Local guest shutdowns: %d\n' "${#shutdown_successes[@]}"
printf ' Failed local shutdowns: %d\n' "${#shutdown_failures[@]}"
printf ' Guests still running locally: %d\n' "${#running_final_guests[@]}"
if (( ${#migration_successes[@]} > 0 )); then
printf '\nMigrated guests:\n'
for guest in "${migration_successes[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name maxmem required_storages \
policy_nodes policy_strict policy_rule destination route_note <<< "$guest"
printf ' %-6s %-5s %-24s -> %s\n' \
"$vmid" "$guest_type" "$name" "$destination"
done
else
warn "Local-storage guests were left running."
fi
if (( ${#migration_failures[@]} > 0 )); then
printf '\nFailed migrations (left unchanged and not force-stopped):\n'
for guest in "${migration_failures[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name maxmem required_storages \
policy_nodes policy_strict policy_rule destination route_note <<< "$guest"
printf ' %-6s %-5s %-24s target=%s\n' \
"$vmid" "$guest_type" "$name" "$destination"
done
fi
if (( ${#shutdown_successes[@]} > 0 )); then
printf '\nLocal-storage guests shut down:\n'
for guest in "${shutdown_successes[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name maxmem reason <<< "$guest"
printf ' %-6s %-5s %s\n' "$vmid" "$guest_type" "$name"
done
fi
if (( ${#final_guests[@]} > 0 )); then
printf '\nGuests still assigned to %s:\n' "$LOCAL_NODE"
for guest in "${final_guests[@]}"; do
IFS=$'\x1f' read -r vmid guest_type status name maxmem <<< "$guest"
printf ' %-6s %-5s %-8s %s\n' "$vmid" "$guest_type" "$status" "$name"
done
fi
printf '\nNo guest was force-stopped.\n'
if (( ${#migration_failures[@]} > 0 ||
${#migration_skipped[@]} > 0 ||
${#unroutable_guests[@]} > 0 ||
${#shutdown_failures[@]} > 0 ||
${#running_final_guests[@]} > 0 )); then
return 1
fi
}
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
main "$@"
fi
mapfile -t final_guests < <(get_local_guests)
printf '\nEvacuation summary:\n'
printf ' Migration failures: %d\n' "${#migration_failures[@]}"
printf ' Guests still assigned to %s: %d\n' "$LOCAL_NODE" "${#final_guests[@]}"
if (( ${#migration_failures[@]} > 0 )); then
printf '\nFailed migrations (left unchanged; not shut down automatically):\n'
printf ' %s\n' "${migration_failures[@]}"
fi
if (( ${#final_guests[@]} > 0 )); then
printf '\nGuests still assigned to %s:\n' "$LOCAL_NODE"
printf ' %s\n' "${final_guests[@]}"
fi
printf '\nNo guest was force-stopped.\n'