6.5 配套代码：池化类瓶颈三合一诊断
对应小节：6.5 瓶颈模式库（一）：池化类 一条命令同时检查连接池 / 线程池 / 调度器三种池化瓶颈。
一、三合一诊断脚本
#!/usr/bin/env bash
# tools/diagnose-pools.sh <METRICS_URL> [PID]
#
# 同时检查三种池化瓶颈：
# ① 连接池耗尽
# ② 线程池饥饿
# ③ 协程调度器污染
set -uo pipefail
URL="${1:-http://127.0.0.1:8080/metrics}"
PID="${2:-$(jcmd 2>/dev/null | grep app.jar | awk '{print $1}')}"
M=$(curl -s --max-time 5 "$URL" 2>/dev/null)
[ -z "$M" ] && { echo "❌ 无法获取指标（$URL）"; exit 1; }
get() { echo "$M" | awk -v k="^$1" '$0 ~ k {printf "%.0f", $2; exit}'; }
echo "═══════════════════════════════════════════════════════════════"
echo "池化类瓶颈诊断"
echo "═══════════════════════════════════════════════════════════════"
echo
# ── ① 连接池 ────────────────────────────────────────────────
echo "【① 数据库连接池】"
ACTIVE=$(get "db_pool_active")
IDLE=$(get "db_pool_idle")
PENDING=$(get "db_pool_pending")
TOTAL=$(get "db_pool_total")
echo " active = ${ACTIVE:-n/a}"
echo " idle = ${IDLE:-n/a}"
echo " pending = ${PENDING:-n/a} ← 关键指标"
echo " total = ${TOTAL:-n/a}"
if [ "${PENDING:-0}" -gt 0 ] 2>/dev/null; then
echo " ❌ 存在排队 → 连接池耗尽（模式一）"
echo " 下一步：先查慢查询（不要先加池）"
echo " psql -c \"SELECT calls, mean_exec_time, total_exec_time, left(query,50)"
echo " FROM pg_stat_statements ORDER BY total_exec_time DESC LIMIT 10;\""
elif [ "${ACTIVE:-0}" -ge "${TOTAL:-1}" ] 2>/dev/null && [ "${TOTAL:-0}" -gt 0 ]; then
echo " ⚠️ active == total（池已满但无排队）→ 没有余量，需观察"
else
echo " ✅ 无排队"
fi
echo
# ── ② 线程池 ────────────────────────────────────────────────
echo "【② 线程池 / 队列】"
Q_DEPTH=$(get "executor_queue_depth")
Q_REMAIN=$(get "executor_queue_remaining")
E_ACTIVE=$(get "executor_active")
echo " active = ${E_ACTIVE:-n/a}"
echo " queue_depth = ${Q_DEPTH:-n/a} ← 关键指标"
echo " queue_remaining= ${Q_REMAIN:-n/a}"
if [ "${Q_DEPTH:-0}" -gt 100 ] 2>/dev/null; then
echo " ❌ 队列积压 → 线程池饥饿（模式二）"
echo " 下一步：查线程在忙什么（可能是在等 IO，而不是在算）"
echo " jcmd \$PID Thread.print | grep -c HikariPool"
echo " jcmd \$PID Thread.print | grep -c socketRead"
elif [ "${Q_DEPTH:-0}" -gt 0 ] 2>/dev/null; then
echo " ⚠️ 队列有少量积压"
else
echo " ✅ 队列空"
fi
echo
# ── ③ 协程调度器 ────────────────────────────────────────────
echo "【③ 协程调度器】"
C_ACTIVE=$(get "app_coroutines_active")
echo " active_coroutines = ${C_ACTIVE:-n/a（需埋点）}"
# 关键判据：RUNNABLE 线程数 vs CPU 核数
CORES=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo "?")
echo " CPU 核数 = $CORES"
if [ -n "$PID" ] && [ "$PID" != "" ]; then
RUNNABLE=$(jcmd "$PID" Thread.print 2>/dev/null | grep -c "java.lang.Thread.State: RUNNABLE" || echo "?")
echo " RUNNABLE 线程数 = $RUNNABLE"
CPU_PCT=$(ps -p "$PID" -o %cpu= 2>/dev/null | tr -d ' ' || echo "?")
echo " 进程 CPU = ${CPU_PCT}%"
if [ "$RUNNABLE" = "$CORES" ] 2>/dev/null && [ -n "$CPU_PCT" ]; then
if awk -v c="$CPU_PCT" 'BEGIN{exit !(c < 50)}'; then
echo " ❌ RUNNABLE == 核数 且 CPU < 50%"
echo " → 高度怀疑：Dispatchers.Default 被阻塞调用占满（模式三）"
echo " 验证：asprof -d 60 -e wall -f wall.html $PID"
echo " （CPU 火焰图看不出这个问题！）"
fi
fi
# 额外检查：Default worker 是否都在阻塞
DEFAULT_BLOCKED=$(jcmd "$PID" Thread.print 2>/dev/null | \
awk '/DefaultDispatcher-worker/{f=1} f&&/socketRead|jdbc|HikariPool|Thread.sleep/{c++} /^$/{f=0} END{print c+0}')
[ "${DEFAULT_BLOCKED:-0}" -gt 0 ] && echo " ⚠️ $DEFAULT_BLOCKED 个 Default worker 停在阻塞调用上"
else
echo " （未提供 PID，跳过线程检查）"
fi
echo
# ── 汇总 ────────────────────────────────────────────────────
echo "═══════════════════════════════════════════════════════════════"
echo "判别速查："
echo
echo " 连接池 pending > 0 → 模式一：连接池耗尽"
echo " 线程池队列持续增长 → 模式二：线程池饥饿"
echo " RUNNABLE == 核数 且 CPU 低 → 模式三：调度器污染 ⚠️ 最难查"
echo
echo " 三种模式的共同点：CPU 不高但延迟高（时间花在排队上）"
echo " 三种模式的鉴别：看哪个池的饱和度指标异常"
echo "═══════════════════════════════════════════════════════════════"
二、调度器污染的专项验证
#!/usr/bin/env bash
# tools/verify-dispatcher-pollution.sh <PID>
#
# 专项验证"Dispatchers.Default 被阻塞调用占满"。
# 这是 Kotlin 后端最隐蔽的瓶颈，值得一个专门的脚本。
set -uo pipefail
PID="${1:?usage: verify-dispatcher-pollution.sh <pid>}"
CORES=$(nproc 2>/dev/null || echo "?")
echo "═══════════════════════════════════════════════════════════════"
echo "调度器污染验证（PID=$PID，CPU 核数=$CORES）"
echo "═══════════════════════════════════════════════════════════════"
echo
# ── 证据 1：CPU 利用率（应该很低）────────────────────────────
echo "【证据 1】进程 CPU 利用率"
CPU=$(top -b -n 1 -p "$PID" 2>/dev/null | tail -1 | awk '{print $9}' || echo "?")
echo " CPU = ${CPU}%"
echo " → 如果 < 40%，符合「等待类问题」的特征"
echo
# ── 证据 2：RUNNABLE 线程数 ────────────────────────────────
echo "【证据 2】线程状态统计"
TMP=$(mktemp)
jcmd "$PID" Thread.print > "$TMP" 2>/dev/null
for st in RUNNABLE BLOCKED WAITING TIMED_WAITING; do
C=$(grep -c "java.lang.Thread.State: $st" "$TMP" || echo 0)
printf " %-16s %s\n" "$st" "$C"
done
RUNNABLE=$(grep -c "java.lang.Thread.State: RUNNABLE" "$TMP" || echo 0)
echo " → RUNNABLE($RUNNABLE) 是否等于核数($CORES)？"
[ "$RUNNABLE" = "$CORES" ] && echo " ⚠️ 相等！这是调度器污染的强信号" || echo " 不相等，可能不是这个模式"
echo
# ── 证据 3：Default worker 在做什么 ────────────────────────
echo "【证据 3】DefaultDispatcher-worker 的栈顶"
awk '/^"DefaultDispatcher-worker/{name=$1; inworker=1; next}
inworker && /java.lang.Thread.State:/{state=$0; next}
inworker && /^\s+at /{if (!printed) {print " " name " " state; print " " $0; printed=1}}
/^$/{inworker=0; printed=0}' "$TMP" | head -30
echo
# ── 证据 4：阻塞调用统计 ───────────────────────────────────
echo "【证据 4】阻塞调用计数（所有线程）"
for pat in "socketRead0" "HikariPool.getConnection" "jdbc" "Thread.sleep" "Unsafe.park"; do
C=$(grep -c "$pat" "$TMP" || echo 0)
[ "$C" -gt 0 ] && printf " %-32s %s\n" "$pat" "$C"
done
echo
rm -f "$TMP"
# ── 结论引导 ────────────────────────────────────────────────
echo "═══════════════════════════════════════════════════════════════"
echo "判据组合（三条同时满足 → 基本确认）："
echo " ① CPU 利用率低（< 40%）"
echo " ② RUNNABLE 线程数 == CPU 核数"
echo " ③ 这些 RUNNABLE 线程的栈停在阻塞调用上"
echo
echo "补充验证（更强）："
echo " asprof -d 60 -e wall -f wall.html $PID"
echo " → wall 火焰图会显示 Default worker 全停在阻塞栈"
echo
echo "修复方向："
echo " ① 把阻塞调用切到 Dispatchers.IO"
echo " ② 或用异步驱动（R2DBC 等）"
echo " ③ 并用 Semaphore 限制对下游的并发"
echo "═══════════════════════════════════════════════════════════════"
三、连接池「先查什么」的决策脚本
#!/usr/bin/env bash
# tools/diagnose-pool-cause.sh <PG_CONN>
#
# pending > 0 时，判断根因是"查询慢"还是"池太小"。
set -uo pipefail
PG="${1:-${PG_CONN:-}}"
[ -z "$PG" ] && { echo "❌ 需要 PG_CONN（例如 postgresql://user:pass@host/db）"; exit 1; }
echo "═══════════════════════════════════════════════════════════════"
echo "连接池排队：根因判别"
echo "═══════════════════════════════════════════════════════════════"
echo
# ── 检查 1：是否有慢查询 ───────────────────────────────────
echo "【检查 1】最贵的语句"
psql "$PG" -tAc "
SELECT calls || ' 次 | mean ' || round(mean_exec_time::numeric,1) || 'ms'
|| ' | total ' || round(total_exec_time::numeric,0) || 'ms | '
|| left(regexp_replace(query, '\s+', ' ', 'g'), 50)
FROM pg_stat_statements
WHERE query NOT LIKE '%pg_stat_statements%'
ORDER BY total_exec_time DESC LIMIT 5;" 2>/dev/null | sed 's/^/ /' || echo " （无法查询）"
echo
# ── 检查 2：当前活跃查询 ───────────────────────────────────
echo "【检查 2】当前活跃查询（是否有长事务）"
psql "$PG" -tAc "
SELECT pid || ' | ' || round(extract(epoch from (now()-query_start))::numeric,1) || 's | '
|| COALESCE(wait_event_type,'-') || ' | ' || left(regexp_replace(query,'\s+',' ','g'),50)
FROM pg_stat_activity
WHERE state='active' AND pid <> pg_backend_pid()
ORDER BY query_start LIMIT 10;" 2>/dev/null | sed 's/^/ /' || echo " （无法查询）"
echo
# ── 检查 3：连接使用率 ─────────────────────────────────────
echo "【检查 3】连接使用率"
psql "$PG" -tAc "
SELECT 'used=' || count(*) || ' / max=' || current_setting('max_connections')
|| ' (' || round(100.0*count(*)/current_setting('max_connections')::int,1) || '%)'
FROM pg_stat_activity;" 2>/dev/null | sed 's/^/ /' || echo " （无法查询）"
echo
# ── 结论引导 ────────────────────────────────────────────────
cat <<'EOF'
═══════════════════════════════════════════════════════════════
判别规则：
检查 1 有慢查询（mean 高或 total 大）
→ 根因是【查询慢】，连接被长时间占用
→ 先优化 SQL / 加索引，不要先加池
检查 1 查询都快，检查 3 连接使用率高
→ 根因是【并发量确实高】或【池太小】
→ 用 Little's Law 计算：所需连接 = 峰值QPS × 单次查询P99
检查 2 有长事务（duration 很大）
→ 根因是【事务范围过大】，连接被事务持有
→ 缩小事务范围，或检查是否有忘记提交的事务
三项都正常
→ pending 可能是瞬时的，或者问题不在数据库侧
→ 检查应用侧：是否有连接泄漏？是否有连接未归还？
═══════════════════════════════════════════════════════════════
EOF
四、动手改造
| 改动 | 观察什么 |
|---|---|
在 Lab 6 的 /slow-db 压测期间跑 diagnose-pools.sh |
看它能否识别出「查询次数异常」 |
在 Lab 6 的 /slow-io 压测期间跑 verify-dispatcher-pollution.sh |
三条判据应该同时满足 |
| 故意把连接池调到 2 并压测 | pending 上升，diagnose-pool-cause.sh 会引导你查 SQL（此时 SQL 是快的，所以会指向"池太小"） |
| 把三个脚本接进监控告警的 runbook | 出问题时值班同学能直接跑 |
| 在正常运行时跑一次 | 建立「健康基线」的输出，便于对比 |
五、这段代码的局限
diagnose-pools.sh依赖指标命名规范：如果你的指标名不同（如hikaricp_connections_pending），需要改脚本。- 「RUNNABLE == 核数」只是强信号，不是证据：也可能恰好有 8 个线程在做健康检查。必须配合 wall 火焰图确认。
diagnose-pool-cause.sh需要pg_stat_statements：如果没启用，检查 1 会失败。- 脚本只能给方向，不能给结论：它们的价值在于「不遗漏要检查的项目」，而不是替代分析。
DebugProbes相关的部分需要改代码：生产环境无法临时启用——所以更好的做法是提前埋一个低开销的协程数指标（第 4 节）。