#!/usr/bin/env bash
# ==============================================================================
# 脚本名称: resolve_terminating_pod.sh
# 功能描述: 自动诊断 Node 节点上导致 Pod 卡在 Terminating 的挂载残留与占用进程 (纯只读)
# 使用方式: 在故障 Pod 所在的宿主机以 root 执行: ./resolve_terminating_pod.sh <POD_UID>
# ==============================================================================
set -o pipefail
POD_UID=$1
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
BLUE='\033[0;34m'
PLAIN='\033[0m'
if [ -z "$POD_UID" ]; then
echo -e "${RED}用法: $0 <POD_UID>${PLAIN}"
echo -e "提示: 可通过 kubectl get pod <pod_name> -o jsonpath='{.metadata.uid}' 获取 UID。"
exit 1
fi
echo -e "${BLUE}================================================================${PLAIN}"
echo -e "${BLUE} Pod Terminating 宿主机挂载阻塞自动诊断脚本 ${PLAIN}"
echo -e "${BLUE}================================================================${PLAIN}"
echo -e "${YELLOW}[INFO] 当前追踪的 Pod UID: ${POD_UID}${PLAIN}\n"
# 1. 检查 Kubelet Pod 目录状态
echo -e "${YELLOW}[1/3] 检查宿主机 Kubelet Pod 目录状态...${PLAIN}"
POD_DIR="/var/lib/kubelet/pods/${POD_UID}"
if [ -d "$POD_DIR" ]; then
echo -e " -> ${YELLOW}[INFO] 宿主机仍存在 Pod 数据目录: ${POD_DIR}${PLAIN}"
else
echo -e " -> ${GREEN}[OK] 宿主机已无该 Pod 目录残留 (/var/lib/kubelet/pods/${POD_UID})。${PLAIN}"
echo -e " -> ${GREEN}[结论] 宿主机数据面已清理完毕。若 Pod 仍处于 Terminating，请检查 API Server 中的 metadata.finalizers 字段。${PLAIN}"
exit 0
fi
# 2. 探测挂载点残留
echo -e "\n${YELLOW}[2/3] 检索操作系统挂载表，排查未卸载的数据卷 (Volume)...${PLAIN}"
# 修复了这里的管道符乱码
MOUNT_RES=$(mount | grep "$POD_UID")
if [ -z "$MOUNT_RES" ]; then
echo -e " -> ${GREEN}[OK] 宿主机无该 UID 关联的活跃挂载点残留。${PLAIN}"
else
echo -e " -> ${RED}[ALERT] 检测到残留挂载点，Kubelet umount 卸载操作受阻:${PLAIN}"
echo "$MOUNT_RES" | awk '{print " " $3}'
fi
# 3. 定位占用句柄的进程
echo -e "\n${YELLOW}[3/3] 扫描文件句柄，定位占用挂载目录的进程...${PLAIN}"
# 修复了这里的管道符乱码
LSOF_RES=$(lsof 2>/dev/null | grep "$POD_UID")
if [ -z "$LSOF_RES" ]; then
echo -e " -> ${YELLOW}[WARN] lsof 未检测到常规用户态进程占用。${PLAIN}"
echo -e " -> ${YELLOW}[提示] 若上述残留挂载点为 NFS/Ceph 等网络存储，可能是存储断连导致的内核 D 状态阻塞，建议尝试执行 umount -l <挂载路径> 进行延迟卸载。${PLAIN}"
else
echo -e " -> ${RED}[DANGER] 检测到正在占用挂载目录的进程：${PLAIN}"
echo -e " ${BLUE}进程名称 PID 用户 文件描述符${PLAIN}"
echo "$LSOF_RES" | awk '{printf " %-12s %-8s %-8s %s\n", $1, $2, $3, $4}' | sort | uniq
PIDS=$(echo "$LSOF_RES" | awk '{print $2}' | sort -u | tr '\n' ' ')
echo -e "\n -> ${GREEN}[处置建议] 请核实上述进程用途，确认可终止后执行以下命令释放文件句柄:${PLAIN}"
echo -e " ${YELLOW}kill -9 $PIDS${PLAIN}"
fi
echo -e "\n${BLUE}================================================================${PLAIN}"