MinIO源码深度解读-第13章第1节-Prometheus-指标系统-metrics.go-指标分类与采集架构完整实现-开发者社区

MinIO源码深度解读 - 第13章第1节：Prometheus 指标系统——metrics.go 指标分类与采集架构完整实现

MinIO 原生支持 Prometheus 监控，所有指标都通过 /minio/v2/metrics/ 端点暴露。

本文导航

指标系统概述
指标分类
指标采集架构
核心指标实现
Prometheus 配置

指标系统概述

MinIO 指标架构

Prometheus 指标系统 ├── 指标定义 (Metrics Definition) │ ├── Cluster 指标 │ ├── Bucket 指标 │ ├── API 指标 │ └── System 指标 │ ├── 指标采集 (Metrics Collector) │ ├── 定时采集 │ └── 事件驱动 │ └── 指标暴露 (Metrics Exposure) ├── /minio/v2/metrics/all ├── /minio/v2/metrics/cluster ├── /minio/v2/metrics/bucket └── /minio/v2/metrics/drive

指标类型

类型	说明	示例
Counter	累计值	API 请求总数
Gauge	当前值	在线磁盘数
Histogram	分布值	API 请求延迟
Summary	摘要值	请求延迟百分位

指标分类

核心指标分类

MinIO 指标 ├── Cluster 指标 │ ├── minio_clusters 总集群数 │ ├── minio_nodes 总节点数 │ ├── minio_disks 总磁盘数 │ ├── minio_disks_online 在线磁盘数 │ ├── minio_disks_offline 离线磁盘数 │ └── minio_erasure_sets 纠删集数量 │ ├── Bucket 指标 │ ├── minio_bucket_capacity_usage_bytes 存储容量 │ ├── minio_bucket_objects_total 对象数量 │ ├── minio_bucket_versions_total 版本数量 │ └── minio_bucket_replication_status 复制状态 │ ├── API 指标 │ ├── minio_api_requests_total API 请求总数 │ ├── minio_api_requests_latency_seconds API 请求延迟 │ ├── minio_api_errors_total API 错误总数 │ └── minio_api_requests_size_bytes 请求大小 │ ├── Erasure Code 指标 │ ├── minio_erasure_shards_encoded 编码分片数 │ ├── minio_erasure_shards_decoded 解码分片数 │ └── minio_erasure_decode_duration_seconds 解码延迟 │ ├── System 指标 │ ├── minio_heap_alloc_bytes 堆内存分配 │ ├── minio_gc_duration_seconds GC 延迟 │ └── minio_goroutines Goroutine 数量 │ └── Healing 指标 ├── minio_heal_items_total 自愈项总数 ├── minio_heal_items_failed 自愈失败项 └── minio_heal_duration_seconds 自愈延迟

指标采集架构

指标采集器

// cmd/metrics.go// 指标采集器typeMetricsCollectorstruct{// 指标注册表registry prometheus.Registerer// 收集配置config MetricsConfig// 统计信息stats MetricsStats}// 指标配置typeMetricsConfigstruct{// 采集间隔Interval time.Duration// 采集目标Targets[]string// 是否启用详细采集Detailedbool}// 指标统计typeMetricsStatsstruct{// 总采集次数TotalCollectionsint64// 总错误数TotalErrorsint64// 平均采集时间AvgCollectionTime time.Duration}

指标注册

// cmd/metrics.go// 注册所有指标func(collector*MetricsCollector)RegisterAll(){// 1. 注册集群指标collector.registerClusterMetrics()// 2. 注册 Bucket 指标collector.registerBucketMetrics()// 3. 注册 API 指标collector.registerAPIMetrics()// 4. 注册纠删码指标collector.registerErasureMetrics()// 5. 注册系统指标collector.registerSystemMetrics()// 6. 注册自愈指标collector.registerHealingMetrics()}// 注册集群指标func(collector*MetricsCollector)registerClusterMetrics(){prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name:"minio_clusters",Help:"Total number of clusters",},func()float64{returnfloat64(globalEndpoints.Pools())})prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name:"minio_nodes",Help:"Total number of nodes",},func()float64{returnfloat64(globalEndpoints.EndpointsCount())})prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name:"minio_disks",Help:"Total number of disks",},func()float64{returnfloat64(globalEndpoints.TotalDisks())})prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name:"minio_disks_online",Help:"Number of online disks",},func()float64{returnfloat64(globalEndpoints.OnlineDisks())})prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name:"minio_disks_offline",Help:"Number of offline disks",},func()float64{returnfloat64(globalEndpoints.OfflineDisks())})}// 注册 Bucket 指标func(collector*MetricsCollector)registerBucketMetrics(){prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name:"minio_bucket_capacity_usage_bytes",Help:"Total bucket capacity usage in bytes",},func()float64{usageInfo,_:=getBucketUsageInfo(context.Background(),BucketUsageOpt{})returnfloat64(usageInfo.Usage.Size)})prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name:"minio_bucket_objects_total",Help:"Total number of objects",},func()float64{usageInfo,_:=getBucketUsageInfo(context.Background(),BucketUsageOpt{})returnfloat64(usageInfo.ObjCount)})}// 注册 API 指标func(collector*MetricsCollector)registerAPIMetrics(){prometheus.NewCounterFunc(prometheus.CounterOpts{Name:"minio_api_requests_total",Help:"Total API requests",},func()float64{returnfloat64(globalAPIStats.get().TotalRequests)})prometheus.NewCounterFunc(prometheus.CounterOpts{Name:"minio_api_errors_total",Help:"Total API errors",},func()float64{returnfloat64(globalAPIStats.get().TotalErrors)})prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name:"minio_api_latency_avg_seconds",Help:"Average API latency in seconds",},func()float64{returnfloat64(globalAPIStats.get().AvgLatency)/1e9})prometheus.NewGaugeFunc(prometheus.GaugeOpts{Name:"minio_api_latency_max_seconds",Help:"Maximum API latency in seconds",},func()float64{returnfloat64(globalAPIStats.get().MaxLatency)/1e9})}

核心指标实现

API 指标实现

// cmd/api-statistics.go// API 统计typeAPIStatsstruct{// 总请求数TotalRequestsint64// 总错误数TotalErrorsint64// 总延迟（纳秒）TotalLatencyint64// 最大延迟（纳秒）MaxLatencyint64// API 映射APIMapmap[string]APICount}// API 请求计数typeAPICountstruct{// 操作名称Operationstring// 总请求数Totalint64// 错误数Errorsint64// 总延迟TotalLatencyint64// 最大延迟MaxLatencyint64// 最小延迟MinLatencyint64}// 更新 API 统计func(stats*APIStats)Update(req APIRequest){stats.mu.Lock()deferstats.mu.Unlock()// 1. 更新全局统计stats.TotalRequests++stats.TotalLatency+=req.Latency stats.TotalErrors+=int64(len(req.Errors))// 2. 更新最大延迟ifreq.Latency>stats.MaxLatency{stats.MaxLatency=req.Latency}// 3. 更新 API 级别统计if_,ok:=stats.APIMap[req.Operation];!ok{stats.APIMap[req.Operation]=APICount{Operation:req.Operation,MinLatency:req.Latency,}}apiCount:=stats.APIMap[req.Operation]apiCount.Total++apiCount.TotalLatency+=req.Latencyifreq.Latency>apiCount.MaxLatency{apiCount.MaxLatency=req.Latency}ifreq.Latency<apiCount.MinLatency{apiCount.MinLatency=req.Latency}iflen(req.Errors)>0{apiCount.Errors+=int64(len(req.Errors))}stats.APIMap[req.Operation]=apiCount}// 获取 API 统计func(stats*APIStats)Get()APIStats{stats.mu.RLock()deferstats.mu.RUnlock()returnAPIStats{TotalRequests:stats.TotalRequests,TotalErrors:stats.TotalErrors,TotalLatency:stats.TotalLatency,MaxLatency:stats.MaxLatency,APIMap:stats.APIMap,}}

纠删码指标实现

// cmd/erasure-metrics.go// 纠删码指标typeErasureMetricsstruct{// 编码计数ShardsEncodedint64// 解码计数ShardsDecodedint64// 编码延迟EncodeLatencyint64// 解码延迟DecodeLatencyint64// 失败计数FailedEncodesint64FailedDecodesint64}// 更新纠删码指标func(metrics*ErasureMetrics)UpdateShardEncoded(shardSizeint64,latencyint64){metrics.mu.Lock()defermetrics.mu.Unlock()metrics.ShardsEncoded++metrics.ShardSizeTotal+=shardSize metrics.EncodeLatency+=latency}func(metrics*ErasureMetrics)UpdateShardDecoded(shardSizeint64,latencyint64){metrics.mu.Lock()defermetrics.mu.Unlock()metrics.ShardsDecoded++metrics.ShardSizeTotal+=shardSize metrics.DecodeLatency+=latency}// 导出 Prometheus 指标func(metrics*ErasureMetrics)ExportPrometheus()map[string]interface{}{metrics.mu.RLock()defermetrics.mu.RUnlock()returnmap[string]interface{}{"shards_encoded":metrics.ShardsEncoded,"shards_decoded":metrics.ShardsDecoded,"total_shard_size":metrics.ShardSizeTotal,"avg_encode_latency":metrics.ShardsEncoded>0? metrics.EncodeLatency/metrics.ShardsEncoded:0,"avg_decode_latency":metrics.ShardsDecoded>0? metrics.DecodeLatency/metrics.ShardsDecoded:0,"failed_encodes":metrics.FailedEncodes,"failed_decodes":metrics.FailedDecodes,}}

Prometheus 配置

Prometheus 配置示例

# prometheus.ymlglobal:scrape_interval:15sevaluation_interval:15sscrape_configs:-job_name:'minio_cluster'metrics_path:'/minio/v2/metrics/cluster'static_configs:-targets:['minio1:9000','minio2:9000','minio3:9000']-job_name:'minio_node'metrics_path:'/minio/v2/metrics/node'static_configs:-targets:['minio1:9000','minio2:9000','minio3:9000']-job_name:'minio_bucket'metrics_path:'/minio/v2/metrics/bucket'static_configs:-targets:['minio1:9000','minio2:9000','minio3:9000']-job_name:'minio_drive'metrics_path:'/minio/v2/metrics/drive'static_configs:-targets:['minio1:9000','minio2:9000','minio3:9000']

Grafana Dashboard

{"dashboard":{"title":"MinIO Cluster Overview","panels":[{"title":"Total Objects","expr":"sum(minio_bucket_objects_total)","type":"stat"},{"title":"Storage Capacity","expr":"sum(minio_bucket_capacity_usage_bytes)","type":"stat"},{"title":"API Request Rate","expr":"rate(minio_api_requests_total[5m])","type":"graph"},{"title":"API Latency P99","expr":"histogram_quantile(0.99, rate(minio_api_requests_latency_seconds_bucket[5m]))","type":"graph"},{"title":"Disk Health","expr":"minio_disks - minio_disks_offline","type":"stat"}]}}