#!/usr/bin/env bash
# ==============================================================================
# 脚本名称: check_java_oom.sh
# 功能描述: 节点级 OOMKilled 巡检与存活 Java 进程基础内存状态观测 (纯只读)
# 使用方式: 在 K8S Node 节点以 root 权限执行
# ==============================================================================
set -o pipefail
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
BLUE='\033[0;34m'
PLAIN='\033[0m'
echo -e "${BLUE}================================================================${PLAIN}"
echo -e "${BLUE} Java Pod 内存与 OOM 状态巡检 ${PLAIN}"
echo -e "${BLUE}================================================================${PLAIN}"
# 1. 检查内核 dmesg 中的 OOMKilled 记录
echo -e "\n${YELLOW}[1/3] 检查操作系统内核 OOM Killer 记录 (最近 20 行)...${PLAIN}"
if command -v dmesg &>/dev/null; then
OOM_LOGS=$(dmesg -T 2>/dev/null | grep -iE "Killed process|Out of memory" | tail -n 5)
if [ -n "$OOM_LOGS" ]; then
echo -e "${RED}[WARN] 发现内核曾触发 OOM Killer 强杀进程:${PLAIN}"
echo "$OOM_LOGS"
else
echo -e "${GREEN}[OK] 未在系统近期日志中发现 OOM Killer 活动记录。${PLAIN}"
fi
else
echo -e "${YELLOW}[INFO] 未找到 dmesg 命令，跳过系统 OOM 检查。${PLAIN}"
fi
# 2. 检查 containerd 运行中的 Java 容器
echo -e "\n${YELLOW}[2/3] 扫描本机运行中的 Java 进程状态...${PLAIN}"
if command -v crictl &>/dev/null; then
# 获取运行中的容器
CONTAINERS=$(crictl ps -q)
JAVA_FOUND=0
for cid in $CONTAINERS; do
# 简单查看进程是否包含 java
if crictl exec "$cid" ps -ef 2>/dev/null | grep -v grep | grep -q "java "; then
JAVA_FOUND=1
CNAME=$(crictl inspect "$cid" | grep -m1 '"name"' | cut -d '"' -f 4)
echo -e "${GREEN}[INFO] 发现 Java 容器 [${CNAME}] (ID: ${cid:0:8})${PLAIN}"
# 检查是否有 jstat 工具
if crictl exec "$cid" command -v jstat &>/dev/null; then
JAVA_PID=$(crictl exec "$cid" sh -c 'jps -q | head -n 1' 2>/dev/null)
if [ -n "$JAVA_PID" ]; then
GC_STAT=$(crictl exec "$cid" jstat -gcutil "$JAVA_PID" 2>/dev/null | tail -n 1)
OLD_GEN=$(echo "$GC_STAT" | awk '{print $4}')
FGC_COUNT=$(echo "$GC_STAT" | awk '{print $8}')
echo -e " -> 当前老年代(O)使用率: ${OLD_GEN}% | Full GC 累计次数: ${FGC_COUNT}"
if [ "$(awk -v og="$OLD_GEN" 'BEGIN{print (og > 90) ? 1 : 0}')" -eq 1 ]; then
echo -e " ${RED}[WARN] 老年代使用率 > 90\%，存在严重堆满溢出风险！${PLAIN}"
fi
fi
else
echo -e " ${YELLOW}[WARN] 容器内未安装 jstat 工具 (可能是 JRE 环境)，无法获取 GC 指标。${PLAIN}"
fi
fi
done
if [ "$JAVA_FOUND" -eq 0 ]; then
echo -e "${GREEN}[OK] 本机当前未运行或未探测到基于 Java 启动的业务容器。${PLAIN}"
fi
else
echo -e "${YELLOW}[INFO] 宿主机无 crictl 命令行工具，跳过容器探测。${PLAIN}"
fi
# 3. 宿主机整体内存水位
echo -e "\n${YELLOW}[3/3] 检查宿主机当前内存使用率...${PLAIN}"
MEM_TOTAL=$(free -m | awk 'NR==2{print $2}')
MEM_USED=$(free -m | awk 'NR==2{print $3}')
MEM_PCT=$(awk "BEGIN {printf \"%.1f\", $MEM_USED/$MEM_TOTAL*100}")
if [ "$(awk -v pct="$MEM_PCT" 'BEGIN{print (pct > 85.0) ? 1 : 0}')" -eq 1 ]; then
echo -e "${RED}[WARN] 宿主机内存总使用率达 ${MEM_PCT}\% (${MEM_USED}M/${MEM_TOTAL}M)，节点级 OOM 风险较高。${PLAIN}"
else
echo -e "${GREEN}[OK] 宿主机物理内存水位正常: ${MEM_PCT}\%。${PLAIN}"
fi
echo -e "\n${BLUE}================================================================${PLAIN}"
echo -e "${BLUE}巡检完毕。若老年代持续偏高，请结合文档 SOP 执行 jmap 抓取快照分析。${PLAIN}"
echo -e "${BLUE}================================================================${PLAIN}"