1.1 配套代码：四层指标的埋点骨架

// src/main/kotlin/metrics/MetricsRegistry.kt
package metrics

import com.zaxxer.hikari.HikariDataSource
import io.micrometer.core.instrument.Gauge
import io.micrometer.core.instrument.MeterRegistry
import io.micrometer.core.instrument.binder.jvm.*
import io.micrometer.core.instrument.binder.system.ProcessorMetrics
import io.micrometer.core.instrument.binder.system.FileDescriptorMetrics
import io.micrometer.prometheusmetrics.PrometheusConfig
import io.micrometer.prometheusmetrics.PrometheusMeterRegistry
import java.util.concurrent.ThreadPoolExecutor
import java.util.concurrent.atomic.AtomicInteger

val registry: PrometheusMeterRegistry = PrometheusMeterRegistry(PrometheusConfig.DEFAULT)

fun registerFourLayers(ds: HikariDataSource, executor: ThreadPoolExecutor) {

    // ══════════════════════════════════════════════════════════
    // 第 3 层：资源层 —— 消耗了什么
    // 这些 binder 会自动注册一批指标，不用手写
    // ══════════════════════════════════════════════════════════
    JvmMemoryMetrics().bindTo(registry)      // jvm_memory_used_bytes / max_bytes
    JvmGcMetrics().bindTo(registry)          // jvm_gc_pause_seconds（含分位数直方图）
    JvmThreadMetrics().bindTo(registry)      // jvm_threads_live / states
    ClassLoaderMetrics().bindTo(registry)    // 类加载数量（冷启动相关）
    ProcessorMetrics().bindTo(registry)      // system_cpu_usage / process_cpu_usage
    FileDescriptorMetrics().bindTo(registry) // 文件描述符使用（连接泄漏的第一信号）

    // ══════════════════════════════════════════════════════════
    // 第 4 层：饱和度层 —— 离上限还有多远（最重要的一层）
    // ══════════════════════════════════════════════════════════

    // 数据库连接池：pending 是最关键的指标
    Gauge.builder("db.pool.pending") { ds.hikariPoolMXBean?.threadsAwaitingConnection ?: 0 }
        .description("正在等待连接的线程数（> 0 说明已经排队）")
        .register(registry)

    Gauge.builder("db.pool.active") { ds.hikariPoolMXBean?.activeConnections ?: 0 }
        .description("正在被使用的连接数")
        .register(registry)

    Gauge.builder("db.pool.idle") { ds.hikariPoolMXBean?.idleConnections ?: 0 }
        .register(registry)

    Gauge.builder("db.pool.total") { ds.hikariPoolMXBean?.totalConnections ?: 0 }
        .register(registry)

    // 线程池队列深度：持续非空说明处理不过来
    Gauge.builder("executor.queue.depth") { executor.queue.size.toDouble() }
        .description("线程池等待队列长度")
        .register(registry)

    Gauge.builder("executor.queue.remaining") { executor.queue.remainingCapacity().toDouble() }
        .description("队列剩余容量（接近 0 说明即将开始拒绝）")
        .register(registry)

    Gauge.builder("executor.active") { executor.activeCount.toDouble() }
        .register(registry)
}

// src/main/kotlin/metrics/BusinessMetrics.kt
package metrics

import io.micrometer.core.instrument.Counter
import io.micrometer.core.instrument.MeterRegistry
import io.micrometer.core.instrument.Timer
import java.time.Duration

/**
 * 桶边界必须围绕自己的 SLO 设置。
 * 默认桶在 5~50ms 区间区分度很差，会让 P99 严重失真。
 */
val sloBuckets: List<Duration> = listOf(
    1, 2, 5, 10, 20, 30, 50, 75, 100, 150, 200, 300, 500, 1000, 2000
).map { Duration.ofMillis(it.toLong()) }

class BusinessMetrics(private val registry: MeterRegistry) {

    // ── 业务层 ────────────────────────────────────────────────
    private val requestsTotal = Counter.builder("app.requests.total")
        .description("总请求数（含失败）")
        .register(registry)

    private val requestsFailed = Counter.builder("app.requests.failed")
        .description("失败请求数（含错误与超时）")
        .register(registry)

    // ── 延迟层 ────────────────────────────────────────────────
    private val requestTimer = Timer.builder("app.request.duration")
        .description("请求耗时")
        .publishPercentileHistogram(true)                     // 关键：发布直方图
        .serviceLevelObjectives(*sloBuckets.toTypedArray())   // 关键：自定义桶边界
        .minimumExpectedValue(Duration.ofMillis(1))
        .maximumExpectedValue(Duration.ofSeconds(5))
        .register(registry)

    fun recordSuccess() = requestsTotal.increment()
    fun recordFailure() = requestsTotal.increment().also { requestsFailed.increment() }
    fun timer(): Timer = requestTimer
}

// src/main/kotlin/Application.kt
package app

import io.ktor.server.application.*
import io.ktor.server.metrics.micrometer.*
import io.ktor.server.response.*
import io.ktor.server.routing.*
import io.micrometer.core.instrument.distribution.DistributionStatisticConfig
import metrics.registry
import metrics.sloBuckets

fun Application.module() {
    install(MicrometerMetrics) {
        registry = metrics.registry
        // 全站统一使用自定义桶边界，而不是默认桶
        distributionStatisticConfig = DistributionStatisticConfig.builder()
            .percentilesHistogram(true)
            .serviceLevelObjectives(*sloBuckets.toTypedArray())
            .minimumExpectedValue(java.time.Duration.ofMillis(1))
            .maximumExpectedValue(java.time.Duration.ofSeconds(5))
            .build()
    }

    routing {
        get("/metrics") {
            call.respondText(metrics.registry.scrape())   // Prometheus 抓取端点
        }
    }
}

// ❌ 灾难：每个用户 ID 一个时间序列，Prometheus 内存会爆
registry.counter("request", "userId", userId)

// ❌ 同样糟糕：URL 全路径（含 ID）
registry.counter("request", "uri", "/orders/12345")

// ✅ 正确：用路由模板 + 可枚举维度
registry.counter("request", "uri", "/orders/{id}", "status", "200", "method", "GET")