第9讲：大模型应用监控与可观测性——OpenTelemetry + Prometheus + Grafana 全链路追踪-开发者社区

一、开篇：为什么 LLM 应用的可观测性比传统服务更重要？

2026 年，LLM 应用在生产环境中面临三个传统服务没有的监控难题：

问题	传统服务	LLM 应用
成本波动大	硬件成本相对固定	一次回答可能花 0.001到1，相差千倍
质量不可预测	输出确定性强	同样的输入可能输出不同质量的结果
故障模式复杂	通常是 5xx 错误	可能是"回答了但不准确"、"幻觉"、"拒绝回答"
依赖链长	通常 2-3 层	LLM → Embedding → 向量库 → Reranker → 缓存，5-6 层

LLM 应用的可观测性 = 传统监控（延迟/错误/吞吐）+ 质量监控 + 成本监控 + 行为监控

二、监控体系总览

┌─────────────────────────────────────────────────────────────────┐ │ LLM 可观测性体系 │ │ │ │ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐ │ │ │ Metrics │ │ Tracing │ │ Logging │ │ │ │ (Prometheus)│ │ (Jaeger) │ │ (Loki) │ │ │ ├─────────────┤ ├─────────────┤ ├─────────────┤ │ │ │ • QPS │ │ • 完整链路 │ │ • 请求日志 │ │ │ │ • 延迟 P50 │ │ • Span 明细 │ │ • 错误日志 │ │ │ │ • 延迟 P99 │ │ • 依赖分析 │ │ • 审计日志 │ │ │ │ • Token 用量│ │ • 瓶颈定位 │ │ • 成本日志 │ │ │ │ • 成本/小时 │ │ • 异常追踪 │ │ • 质量采样 │ │ │ │ • 缓存命中 │ │ │ │ │ │ │ │ • 模型分布 │ │ │ │ │ │ │ └─────────────┘ └─────────────┘ └─────────────┘ │ │ │ │ ┌──────────────────────────────────────────────────────────┐ │ │ │ 告警规则 │ │ │ │ ┌──────────┐ ┌──────────┐ ┌──────────┐ ┌──────────┐ │ │ │ │ │ 成本突增 │ │ 延迟飙高 │ │ 错误率 │ │ 召回率低 │ │ │ │ │ └──────────┘ └──────────┘ └──────────┘ └──────────┘ │ │ │ └──────────────────────────────────────────────────────────┘ │ │ │ │ ┌──────────────────────────────────────────────────────────┐ │ │ │ 可视化面板（Grafana） │ │ │ │ 实时大盘 | 成本看板 | 质量看板 | 链路追踪 | 告警中心 │ │ │ └──────────────────────────────────────────────────────────┘ │ └─────────────────────────────────────────────────────────────────┘

三、核心数据模型

// observability/models.go package observability import ( "context" "time" ) // LLMSpan LLM 调用 Span type LLMSpan struct { TraceID string `json:"trace_id"` SpanID string `json:"span_id"` ParentSpanID string `json:"parent_span_id"` Name string `json:"name"` // 操作名称 Kind string `json:"kind"` // internal, client, server // 时间 StartTime time.Time `json:"start_time"` EndTime time.Time `json:"end_time"` DurationMs int64 `json:"duration_ms"` // LLM 特定属性 ModelName string `json:"model_name"` Provider string `json:"provider"` RequestType string `json:"request_type"` // chat, embedding, rerank // Token 用量 PromptTokens int `json:"prompt_tokens"` CompletionTokens int `json:"completion_tokens"` TotalTokens int `json:"total_tokens"` // 成本 CostUSD float64 `json:"cost_usd"` // 状态 Status string `json:"status"` // ok, error, timeout ErrorMessage string `json:"error_message,omitempty"` // 元数据 UserID string `json:"user_id"` SessionID string `json:"session_id"` Tags map[string]string `json:"tags"` // 缓存 FromCache bool `json:"from_cache"` CacheHit bool `json:"cache_hit"` } // LLMMetric LLM 指标 type LLMMetric struct { Timestamp time.Time `json:"timestamp"` // 计数指标 RequestCount int64 `json:"request_count"` ErrorCount int64 `json:"error_count"` TimeoutCount int64 `json:"timeout_count"` // 延迟指标 P50LatencyMs float64 `json:"p50_latency_ms"` P95LatencyMs float64 `json:"p95_latency_ms"` P99LatencyMs float64 `json:"p99_latency_ms"` // Token 指标 TotalPromptTokens int64 `json:"total_prompt_tokens"` TotalCompletionTokens int64 `json:"total_completion_tokens"` TotalTokens int64 `json:"total_tokens"` // 成本指标 TotalCostUSD float64 `json:"total_cost_usd"` // 缓存指标 CacheHitCount int64 `json:"cache_hit_count"` CacheMissCount int64 `json:"cache_miss_count"` CacheHitRate float64 `json:"cache_hit_rate"` // 模型分布 ModelDistribution map[string]int64 `json:"model_distribution"` // 重排序指标 RerankGain float64 `json:"rerank_gain"` // 重排序带来的精度提升 } // AlertRule 告警规则 type AlertRule struct { Name string `json:"name"` Description string `json:"description"` Metric string `json:"metric"` Condition string `json:"condition"` // gt, lt, eq, change_rate Threshold float64 `json:"threshold"` Duration time.Duration `json:"duration"` // 持续多久触发 Severity string `json:"severity"` // critical, warning, info Channels []string `json:"channels"` // email, slack, wechat }

四、OpenTelemetry 埋点 SDK

// observability/tracer.go package observability import ( "context" "fmt" "runtime" "sync" "time" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" ) // LLMTracer LLM 专用 Tracer type LLMTracer struct { tracer trace.Tracer metrics *MetricsCollector logger *LogCollector config TracerConfig } type TracerConfig struct { ServiceName string `json:"service_name"` Environment string `json:"environment"` // dev, staging, prod SampleRate float64 `json:"sample_rate"` // 采样率 0.0-1.0 EnableDetailed bool `json:"enable_detailed"` // 是否记录详细属性 CostAlertThreshold float64 `json:"cost_alert_threshold"` // 单次调用成本告警阈值 } func NewLLMTracer(config TracerConfig) *LLMTracer { return &LLMTracer{ tracer: otel.Tracer("llm-tracer"), metrics: NewMetricsCollector(config.ServiceName), logger: NewLogCollector(config.ServiceName, config.Environment), config: config, } } // StartLLMSpan 开始一个 LLM 调用 Span func (t *LLMTracer) StartLLMSpan(ctx context.Context, operation string, attrs ...attribute.KeyValue) (context.Context, *LLMSpan) { ctx, span := t.tracer.Start(ctx, operation, trace.WithAttributes(attrs...), trace.WithSpanKind(trace.SpanKindClient), ) llmSpan := &LLMSpan{ TraceID: span.SpanContext().TraceID().String(), SpanID: span.SpanContext().SpanID().String(), Name: operation, StartTime: time.Now(), Tags: make(map[string]string), } return ctx, llmSpan } // EndLLMSpan 结束一个 LLM 调用 Span func (t *LLMTracer) EndLLMSpan(span *LLMSpan) { span.EndTime = time.Now() span.DurationMs = time.Since(span.StartTime).Milliseconds() // 记录指标 t.metrics.RecordLLMCall(span) // 记录日志 t.logger.LogLLMCall(span) // 检查成本告警 if span.CostUSD > t.config.CostAlertThreshold { t.logger.Warn("High cost LLM call", "cost", span.CostUSD, "model", span.ModelName, "trace_id", span.TraceID, ) } } // RecordTokenUsage 记录 Token 用量 func (t *LLMTracer) RecordTokenUsage(span *LLMSpan, promptTokens, completionTokens int) { span.PromptTokens = promptTokens span.CompletionTokens = completionTokens span.TotalTokens = promptTokens + completionTokens // 估算成本（简化版） rates := map[string][2]float64{ "deepseek-chat": {0.014, 0.028}, "gpt-4o-mini": {0.015, 0.060}, "claude-3-haiku": {0.025, 0.125}, } if rate, ok := rates[span.ModelName]; ok { span.CostUSD = (float64(promptTokens)/1000.0)*rate[0] + (float64(completionTokens)/1000.0)*rate[1] } } // RecordError 记录错误 func (t *LLMTracer) RecordError(span *LLMSpan, err error) { span.Status = "error" span.ErrorMessage = err.Error() } // shouldSample 判断是否采样 func (t *LLMTracer) shouldSample() bool { // 生产环境按采样率，开发环境全量 if t.config.Environment == "prod" { return fastRand() < t.config.SampleRate } return true } // fastRand 快速随机数（简化版） func fastRand() float64 { return float64(time.Now().UnixNano()%10000) / 10000.0 } // ========== 指标收集器 ========== // MetricsCollector 指标收集器 type MetricsCollector struct { serviceName string counters map[string]*Counter histograms map[string]*Histogram gauges map[string]*Gauge mu sync.RWMutex } func NewMetricsCollector(serviceName string) *MetricsCollector { return &MetricsCollector{ serviceName: serviceName, counters: make(map[string]*Counter), histograms: make(map[string]*Histogram), gauges: make(map[string]*Gauge), } } // RecordLLMCall 记录 LLM 调用指标 func (mc *MetricsCollector) RecordLLMCall(span *LLMSpan) { mc.mu.Lock() defer mc.mu.Unlock() // 请求计数 mc.getCounter("llm.requests.total").Add(1) mc.getCounter(fmt.Sprintf("llm.requests.model.%s", span.ModelName)).Add(1) if span.Status == "error" { mc.getCounter("llm.errors.total").Add(1) } // Token 计数 mc.getCounter("llm.tokens.prompt").Add(int64(span.PromptTokens)) mc.getCounter("llm.tokens.completion").Add(int64(span.CompletionTokens)) mc.getCounter("llm.tokens.total").Add(int64(span.TotalTokens)) // 成本 mc.getCounter("llm.cost.usd").Add(int64(span.CostUSD * 1000000)) // 微美元 // 延迟直方图 mc.getHistogram("llm.latency.ms").Observe(float64(span.DurationMs)) // 缓存 if span.FromCache { mc.getCounter("llm.cache.hit").Add(1) } else { mc.getCounter("llm.cache.miss").Add(1) } } func (mc *MetricsCollector) getCounter(name string) *Counter { if c, ok := mc.counters[name]; ok { return c } c := &Counter{Name: name} mc.counters[name] = c return c } func (mc *MetricsCollector) getHistogram(name string) *Histogram { if h, ok := mc.histograms[name]; ok { return h } h := &Histogram{Name: name, Buckets: []float64{100, 300, 500, 1000, 2000, 5000, 10000}} mc.histograms[name] = h return h } // Export 导出指标（供 Prometheus 拉取） func (mc *MetricsCollector) Export() string { mc.mu.RLock() defer mc.mu.RUnlock() output := "" for _, c := range mc.counters { output += c.Format() } for _, h := range mc.histograms { output += h.Format() } return output } // Counter 计数器 type Counter struct { Name string Value int64 } func (c *Counter) Add(v int64) { c.Value += v } func (c *Counter) Format() string { return fmt.Sprintf("# HELP %s LLM metric counter\n# TYPE %s counter\n%s %d\n", c.Name, c.Name, c.Name, c.Value) } // Histogram 直方图 type Histogram struct { Name string Buckets []float64 Counts []int64 Sum float64 Count int64 } func (h *Histogram) Observe(v float64) { h.Sum += v h.Count++ for i, b := range h.Buckets { if v <= b { if len(h.Counts) <= i { h.Counts = append(h.Counts, make([]int64, i+1)...) } h.Counts[i]++ break } } } func (h *Histogram) Format() string { output := fmt.Sprintf("# HELP %s LLM metric histogram\n# TYPE %s histogram\n", h.Name, h.Name) for i, b := range h.Buckets { le := "+Inf" if i < len(h.Buckets) { le = fmt.Sprintf("%.0f", b) } count := int64(0) if i < len(h.Counts) { count = h.Counts[i] } output += fmt.Sprintf("%s_bucket{le=\"%s\"} %d\n", h.Name, le, count) } output += fmt.Sprintf("%s_sum %f\n%s_count %d\n", h.Name, h.Sum, h.Name, h.Count) return output } // Gauge 仪表盘 type Gauge struct { Name string Value float64 } // ========== 日志收集器 ========== // LogCollector 日志收集器 type LogCollector struct { serviceName string environment string } func NewLogCollector(serviceName, environment string) *LogCollector { return &LogCollector{ serviceName: serviceName, environment: environment, } } func (lc *LogCollector) LogLLMCall(span *LLMSpan) { // 生产环境：写入 Loki / Elasticsearch level := "INFO" if span.Status == "error" { level = "ERROR" } logEntry := fmt.Sprintf("[%s] [%s] model=%s duration=%dms tokens=%d cost=$%.4f status=%s trace=%s", level, lc.serviceName, span.ModelName, span.DurationMs, span.TotalTokens, span.CostUSD, span.Status, span.TraceID, ) fmt.Println(logEntry) } func (lc *LogCollector) Warn(msg string, keysAndValues ...interface{}) { args := "" for i := 0; i < len(keysAndValues); i += 2 { if i+1 < len(keysAndValues) { args += fmt.Sprintf(" %v=%v", keysAndValues[i], keysAndValues[i+1]) } } fmt.Printf("[WARN] [%s] %s%s\n", lc.serviceName, msg, args) }

五、RAG Pipeline 全链路追踪

// observability/rag_tracer.go package observability import ( "context" "fmt" "time" ) // RAGTracer RAG 链路追踪器 type RAGTracer struct { tracer *LLMTracer } func NewRAGTracer(config TracerConfig) *RAGTracer { return &RAGTracer{ tracer: NewLLMTracer(config), } } // TraceRAGPipeline 追踪完整的 RAG 流水线 func (rt *RAGTracer) TraceRAGPipeline(ctx context.Context, query string, pipelineFn func(context.Context) (*RAGTraceResult, error)) (*RAGTraceResult, error) { ctx, rootSpan := rt.tracer.StartLLMSpan(ctx, "rag.pipeline", attribute.String("query", truncateStr(query, 100)), attribute.String("service", rt.tracer.config.ServiceName), ) rootSpan.Kind = "server" rootSpan.RequestType = "rag_pipeline" start := time.Now() result, err := pipelineFn(ctx) duration := time.Since(start) if err != nil { rt.tracer.RecordError(rootSpan, err) } rootSpan.DurationMs = duration.Milliseconds() if result != nil { rootSpan.TotalTokens = result.TotalTokens rootSpan.CostUSD = result.TotalCost rootSpan.Tags["retrieved_docs"] = fmt.Sprintf("%d", result.RetrievedDocs) rootSpan.Tags["used_docs"] = fmt.Sprintf("%d", result.UsedDocs) rootSpan.Tags["cache_hit"] = fmt.Sprintf("%v", result.FromCache) } rt.tracer.EndLLMSpan(rootSpan) return result, err } // TraceLLMCall 追踪 LLM 调用 func (rt *RAGTracer) TraceLLMCall(ctx context.Context, model string, callFn func(context.Context) (promptTokens, completionTokens int, err error)) (int, int, error) { ctx, span := rt.tracer.StartLLMSpan(ctx, fmt.Sprintf("llm.%s", model), attribute.String("model", model), ) span.ModelName = model span.RequestType = "chat" promptTokens, completionTokens, err := callFn(ctx) rt.tracer.RecordTokenUsage(span, promptTokens, completionTokens) if err != nil { rt.tracer.RecordError(span, err) } rt.tracer.EndLLMSpan(span) return promptTokens, completionTokens, err } // TraceEmbedding 追踪 Embedding 调用 func (rt *RAGTracer) TraceEmbedding(ctx context.Context, model string, textLen int, callFn func(context.Context) error) error { ctx, span := rt.tracer.StartLLMSpan(ctx, fmt.Sprintf("embedding.%s", model), attribute.String("model", model), attribute.Int("text_length", textLen), ) span.ModelName = model span.RequestType = "embedding" err := callFn(ctx) if err != nil { rt.tracer.RecordError(span, err) } rt.tracer.EndLLMSpan(span) return err } // TraceVectorSearch 追踪向量检索 func (rt *RAGTracer) TraceVectorSearch(ctx context.Context, indexName string, topK int, callFn func(context.Context) (int, error)) (int, error) { ctx, span := rt.tracer.StartLLMSpan(ctx, fmt.Sprintf("vector_search.%s", indexName), attribute.String("index", indexName), attribute.Int("top_k", topK), ) span.RequestType = "vector_search" resultCount, err := callFn(ctx) span.Tags["result_count"] = fmt.Sprintf("%d", resultCount) if err != nil { rt.tracer.RecordError(span, err) } rt.tracer.EndLLMSpan(span) return resultCount, err } // TraceRerank 追踪重排序 func (rt *RAGTracer) TraceRerank(ctx context.Context, candidateCount int, callFn func(context.Context) (selectedCount int, gain float64, err error)) (int, float64, error) { ctx, span := rt.tracer.StartLLMSpan(ctx, "rerank", attribute.Int("candidates", candidateCount), ) span.RequestType = "rerank" selectedCount, gain, err := callFn(ctx) span.Tags["selected"] = fmt.Sprintf("%d", selectedCount) span.Tags["gain"] = fmt.Sprintf("%.2f", gain) if err != nil { rt.tracer.RecordError(span, err) } rt.tracer.EndLLMSpan(span) return selectedCount, gain, err } // RAGTraceResult RAG 追踪结果 type RAGTraceResult struct { TotalTokens int TotalCost float64 RetrievedDocs int UsedDocs int FromCache bool DurationMs int64 } func truncateStr(s string, maxLen int) string { runes := []rune(s) if len(runes) > maxLen { return string(runes[:maxLen]) + "..." } return s }

六、Prometheus 指标暴露

// observability/exporter.go package observability import ( "fmt" "net/http" "sync" ) // MetricsExporter Prometheus 指标导出器 type MetricsExporter struct { collectors map[string]*MetricsCollector mu sync.RWMutex } var globalExporter *MetricsExporter var once sync.Once func GetGlobalExporter() *MetricsExporter { once.Do(func() { globalExporter = &MetricsExporter{ collectors: make(map[string]*MetricsCollector), } }) return globalExporter } // RegisterCollector 注册指标收集器 func (me *MetricsExporter) RegisterCollector(name string, collector *MetricsCollector) { me.mu.Lock() defer me.mu.Unlock() me.collectors[name] = collector } // ServeHTTP 实现 http.Handler，供 Prometheus 拉取 func (me *MetricsExporter) ServeHTTP(w http.ResponseWriter, r *http.Request) { me.mu.RLock() defer me.mu.RUnlock() w.Header().Set("Content-Type", "text/plain; charset=utf-8") for name, collector := range me.collectors { fmt.Fprintf(w, "# HELP %s LLM application metrics\n# TYPE %s gauge\n", name, name) fmt.Fprint(w, collector.Export()) } } // StartMetricsServer 启动指标 HTTP 服务 func StartMetricsServer(addr string) error { exporter := GetGlobalExporter() mux := http.NewServeMux() mux.Handle("/metrics", exporter) return http.ListenAndServe(addr, mux) }

七、告警规则

// observability/alerts.go package observability import ( "fmt" "time" ) // AlertManager 告警管理器 type AlertManager struct { rules []AlertRule history map[string][]time.Time // rule name -> trigger times } func NewAlertManager() *AlertManager { return &AlertManager{ rules: DefaultAlertRules(), history: make(map[string][]time.Time), } } // DefaultAlertRules 默认告警规则 func DefaultAlertRules() []AlertRule { return []AlertRule{ { Name: "high_cost_per_request", Description: "单次 LLM 调用成本超过 $0.05", Metric: "llm.cost.per_request", Condition: "gt", Threshold: 0.05, Duration: 1 * time.Minute, Severity: "warning", Channels: []string{"slack"}, }, { Name: "high_error_rate", Description: "LLM 调用错误率超过 5%", Metric: "llm.error_rate", Condition: "gt", Threshold: 0.05, Duration: 5 * time.Minute, Severity: "critical", Channels: []string{"slack", "email"}, }, { Name: "high_latency_p99", Description: "P99 延迟超过 10 秒", Metric: "llm.latency.p99", Condition: "gt", Threshold: 10000, Duration: 5 * time.Minute, Severity: "warning", Channels: []string{"slack"}, }, { Name: "cost_spike", Description: "小时级成本突增超过 200%", Metric: "llm.cost.hourly", Condition: "change_rate", Threshold: 2.0, Duration: 10 * time.Minute, Severity: "critical", Channels: []string{"slack", "wechat"}, }, { Name: "low_cache_hit_rate", Description: "缓存命中率低于 20%", Metric: "llm.cache.hit_rate", Condition: "lt", Threshold: 0.2, Duration: 30 * time.Minute, Severity: "info", Channels: []string{"slack"}, }, { Name: "zero_recall", Description: "检索召回率为 0", Metric: "rag.recall_rate", Condition: "eq", Threshold: 0, Duration: 5 * time.Minute, Severity: "critical", Channels: []string{"slack", "email"}, }, { Name: "budget_exhausted", Description: "月度预算使用超过 80%", Metric: "llm.cost.monthly_budget_usage", Condition: "gt", Threshold: 0.8, Duration: 1 * time.Hour, Severity: "warning", Channels: []string{"email"}, }, } } // Evaluate 评估告警规则 func (am *AlertManager) Evaluate(metrics map[string]float64) []AlertRule { var triggered []AlertRule for _, rule := range am.rules { value, ok := metrics[rule.Metric] if !ok { continue } if am.evaluateCondition(rule.Condition, value, rule.Threshold) { triggered = append(triggered, rule) } } return triggered } func (am *AlertManager) evaluateCondition(condition string, value, threshold float64) bool { switch condition { case "gt": return value > threshold case "lt": return value < threshold case "eq": return value == threshold case "change_rate": return value > threshold default: return false } } // SendAlert 发送告警 func (am *AlertManager) SendAlert(rule AlertRule, currentValue float64) { message := fmt.Sprintf("[%s] %s\n当前值: %.2f\n阈值: %.2f\n描述: %s", rule.Severity, rule.Name, currentValue, rule.Threshold, rule.Description) for _, channel := range rule.Channels { switch channel { case "slack": sendSlackAlert(message) case "email": sendEmailAlert(message) case "wechat": sendWechatAlert(message) } } } func sendSlackAlert(message string) { fmt.Printf("[SLACK] %s\n", message) } func sendEmailAlert(message string) { fmt.Printf("[EMAIL] %s\n", message) } func sendWechatAlert(message string) { fmt.Printf("[WECHAT] %s\n", message) }

八、Grafana Dashboard 配置

{ "dashboard": { "title": "LLM Application Monitoring", "panels": [ { "title": "Request Volume & Error Rate", "type": "timeseries", "targets": [ { "expr": "rate(llm_requests_total[5m])", "legendFormat": "Total Requests" }, { "expr": "rate(llm_errors_total[5m])", "legendFormat": "Errors" } ] }, { "title": "Latency (P50/P95/P99)", "type": "timeseries", "targets": [ { "expr": "histogram_quantile(0.50, sum(rate(llm_latency_ms_bucket[5m])) by (le))", "legendFormat": "P50" }, { "expr": "histogram_quantile(0.95, sum(rate(llm_latency_ms_bucket[5m])) by (le))", "legendFormat": "P95" }, { "expr": "histogram_quantile(0.99, sum(rate(llm_latency_ms_bucket[5m])) by (le))", "legendFormat": "P99" } ] }, { "title": "Token Consumption", "type": "timeseries", "targets": [ { "expr": "rate(llm_tokens_total[5m])", "legendFormat": "Total Tokens" }, { "expr": "rate(llm_tokens_prompt[5m])", "legendFormat": "Prompt Tokens" }, { "expr": "rate(llm_tokens_completion[5m])", "legendFormat": "Completion Tokens" } ] }, { "title": "Cost (USD/hour)", "type": "timeseries", "targets": [ { "expr": "rate(llm_cost_usd[1h])", "legendFormat": "Cost per hour" } ] }, { "title": "Cache Hit Rate", "type": "timeseries", "targets": [ { "expr": "rate(llm_cache_hit[5m]) / (rate(llm_cache_hit[5m]) + rate(llm_cache_miss[5m]))", "legendFormat": "Hit Rate" } ] }, { "title": "Model Distribution", "type": "piechart", "targets": [ { "expr": "sum by(model) (rate(llm_requests_model[5m]))", "legendFormat": "{{model}}" } ] } ] } }

九、集成到 RAG Pipeline

// example/integration.go package main import ( "context" "fmt" "observability" "rag" ) func main() { // 1. 创建追踪器 tracer := observability.NewRAGTracer(observability.TracerConfig{ ServiceName: "rag-service", Environment: "prod", SampleRate: 0.1, // 生产环境 10% 采样 EnableDetailed: true, CostAlertThreshold: 0.05, // 单次超过 $0.05 告警 }) // 2. 启动指标服务 go func() { exporter := observability.GetGlobalExporter() exporter.RegisterCollector("rag", tracer.Tracer().MetricsCollector()) observability.StartMetricsServer(":9090") }() // 3. 创建 RAG Pipeline（带追踪） pipeline := rag.NewRAGPipeline( &TracedLLM{tracer: tracer}, &TracedEmbedder{tracer: tracer}, &TracedVectorIndex{tracer: tracer}, &TracedBM25Index{tracer: tracer}, nil, rag.DefaultPipelineConfig(), ) // 4. 执行带追踪的查询 ctx := context.Background() result, err := tracer.TraceRAGPipeline(ctx, "如何使用 Go 实现 RAG？", func(ctx context.Context) (*observability.RAGTraceResult, error) { ragResult, err := pipeline.Run(ctx, "如何使用 Go 实现 RAG？") if err != nil { return nil, err } return &observability.RAGTraceResult{ TotalTokens: ragResult.Metrics.ContextTokens + estimateResponseTokens(ragResult.Answer), TotalCost: 0.02, // 实际从 LLM 调用中获取 RetrievedDocs: ragResult.Metrics.RetrievedDocs, UsedDocs: ragResult.Metrics.UsedDocs, DurationMs: ragResult.Metrics.TotalDurationMs, }, nil }) if err != nil { fmt.Printf("Error: %v\n", err) return } fmt.Printf("Result: %s\n", result) } // TracedLLM 带追踪的 LLM 客户端 type TracedLLM struct { tracer *observability.RAGTracer } func (l *TracedLLM) Chat(ctx context.Context, messages []rag.Message) (string, error) { var response string _, _, err := l.tracer.TraceLLMCall(ctx, "deepseek-chat", func(ctx context.Context) (int, int, error) { // 实际的 LLM 调用 response = "模拟回复" return 100, 50, nil // prompt_tokens, completion_tokens }) return response, err } // TracedEmbedder 带追踪的 Embedder type TracedEmbedder struct { tracer *observability.RAGTracer } func (e *TracedEmbedder) Embed(ctx context.Context, text string) ([]float32, error) { var emb []float32 err := e.tracer.TraceEmbedding(ctx, "text-embedding-3", len(text), func(ctx context.Context) error { emb = make([]float32, 768) return nil }) return emb, err } // TracedVectorIndex 带追踪的向量索引 type TracedVectorIndex struct { tracer *observability.RAGTracer } func (idx *TracedVectorIndex) Search(ctx context.Context, embedding []float32, topK int, filters map[string]interface{}) ([]rag.Document, error) { var docs []rag.Document _, err := idx.tracer.TraceVectorSearch(ctx, "milvus-demo", topK, func(ctx context.Context) (int, error) { docs = []rag.Document{{ID: "1", Content: "模拟文档", Score: 0.95}} return len(docs), nil }) return docs, err } // TracedBM25Index 带追踪的 BM25 索引 type TracedBM25Index struct { tracer *observability.RAGTracer } func (idx *TracedBM25Index) Search(ctx context.Context, query string, topK int) ([]rag.Document, error) { return []rag.Document{}, nil } func estimateResponseTokens(answer string) int { return len([]rune(answer)) / 2 }

十、关键要点

LLM 应用的三维监控：Metrics + Tracing + Logging — 三者缺一不可
采样率是成本控制的关键 — 生产环境 10% 采样足够发现问题，全量采样成本过高
Token 和成本是 LLM 特有的核心指标 — 传统监控不关心这些，但 LLM 应用必须关注
全链路追踪是调试利器 — 一次用户请求经过 5-6 个组件，没有 Trace 根本找不到瓶颈
告警要有分级 — Critical 立即通知，Warning 白天处理，Info 周报汇总
成本告警最重要 — 一次配置错误可能导致小时级 $1000+ 的损失
Grafana 面板要分层 — 运维看实时大盘，管理者看成本看板，开发者看链路追踪

十一、预告

第 10 讲预告：《大模型应用测试：从单元测试到 E2E 评测的完整体系》
你将学到：
LLM 应用的测试为什么难？确定性 vs 非确定性输出的挑战
单元测试：Prompt 模板测试、工具调用测试、输出格式验证
集成测试：RAG Pipeline 端到端测试、多 Agent 协作测试
评估体系：BLEU/ROUGE/BERTScore、LLM-as-Judge、人工评估
回归测试：Diff 检测、语义等价判断、自动化 CI/CD 集成
Go 实现的测试框架