# 会议维度
video.meeting.id: "string"           # 全局唯一会议ID
video.meeting.mode: "sfu|mcu|p2p"    # 架构模式
video.meeting.participant_count: "int" # 实时人数

# 参会者/设备维度
video.participant.id: "string"
video.participant.role: "host|guest|audience"
video.device.model: "string"         # 如 iPhone 15, MacBook Pro 14
video.device.os_version: "string"
video.network.type: "wifi|4g|5g|ethernet"
video.network.rtt_ms: "int"          # 客户端上报的实时 RTT

# 媒体流维度 (关键：用于关联上下行质量)
video.track.id: "string"             # MediaStreamTrack ID
video.track.kind: "video|audio"
video.track.codec: "h264|vp8|vp9|opus"
video.track.spatial_layer: "int"     # SVC 空间层
video.track.temporal_layer: "int"    # SVC 时间层

# 质量体验维度 (QoE)
video.qoe.freeze_rate: "double"      # 卡顿率
video.qoe.mos_score: "double"        # MOS 评分
video.qoe.first_frame_delay_ms: "int" # 首帧渲染延迟

// pkg/tracing/otel.go
import (
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/attribute"
    "go.opentelemetry.io/otel/trace"
    "go.opentelemetry.io/otel/metric"
)

var (
    tracer  = otel.Tracer("company/media-server/sfu")
    meter   = otel.Meter("company/media-server/sfu")
    
    // 预创建指标工具，避免热路径分配
    forwardDurationHistogram metric.Float64Histogram
    bitrateGauge             metric.Float64ObservableGauge
)

func InitMetrics() error {
    var err error
    forwardDurationHistogram, err = meter.Float64Histogram(
        "video.sfu.forward.duration_ms",
        metric.WithDescription("SFU 转发单帧处理耗时"),
        metric.WithUnit("ms"),
    )
    // ... 错误处理
    
    // 可观测 Gauge 回调上报实时码率
    _, err = meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
        // 遍历活跃 Track 采样上报
        sfu.IterateTracks(func(t *Track) {
            attrs := []attribute.KeyValue{
                attribute.String("video.track.id", t.ID),
                attribute.String("video.track.kind", string(t.Kind)),
                attribute.String("video.meeting.id", t.MeetingID),
            }
            o.ObserveFloat64(bitrateGauge, t.CurrentBitrateKbps, attrs...)
        })
        return nil
    }, bitrateGauge)
    return err
}

// 核心转发路径埋点
func (s *SFU) ForwardPacket(ctx context.Context, track *Track, pkt *rtp.Packet) {
    // 1. 从上下文提取父 SpanContext (由信令建立时注入)
    ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(pkt.Headers))
    
    // 2. 启动 Span (采样决策已在 TracerProvider 配置)
    ctx, span := tracer.Start(ctx, "SFU.ForwardPacket",
        trace.WithAttributes(
            attribute.String("video.meeting.id", track.MeetingID),
            attribute.String("video.track.id", track.ID),
            attribute.Int("video.track.spatial_layer", pkt.SpatialLayer),
            attribute.Int("rtp.sequence_number", int(pkt.SequenceNumber)),
        ),
        trace.WithSpanKind(trace.SpanKindConsumer), // 作为消费者接收
    )
    defer span.End()

    // 3. 业务逻辑处理...
    start := time.Now()
    err := s.router.Route(ctx, track, pkt)
    
    // 4. 记录耗时指标 & 异常
    forwardDurationHistogram.Record(ctx, float64(time.Since(start).Milliseconds()),
        attribute.String("video.meeting.id", track.MeetingID),
        attribute.Bool("error", err != nil),
    )
    if err != nil {
        span.RecordError(err, trace.WithAttributes(attribute.String("error.type", "route_failed")))
        span.SetStatus(codes.Error, err.Error())
    }
}

// sdk/tracing/VideoTrace.ts
import { context, trace, metrics, propagation, SpanStatusCode } from '@opentelemetry/api';
import { W3CTraceContextPropagator } from '@opentelemetry/core';

const tracer = trace.getTracer('company/meeting-web-sdk');
const meter = metrics.getMeter('company/meeting-web-sdk');

// 自定义指标
const firstFrameDelay = meter.createHistogram('video.client.first_frame_delay_ms', { unit: 'ms' });
const freezeDuration = meter.createHistogram('video.client.freeze_duration_ms', { unit: 'ms' });
const joinMeetingDuration = meter.createHistogram('video.client.join_duration_ms', { unit: 'ms' });

// 1. 加入会议全链路入口
export async function joinMeeting(meetingId: string, token: string) {
  const rootSpan = tracer.startSpan('Client.JoinMeeting', {
    attributes: {
      'video.meeting.id': meetingId,
      'video.client.sdk_version': SDK_VERSION,
      'video.device.browser': getBrowserInfo(),
    },
    kind: SpanKind.CLIENT,
  });

  return context.with(trace.setSpan(context.active(), rootSpan), async () => {
    const joinStart = Date.now();
    try {
      // 信令交互 (自动携带 traceparent)
      const signal = await signaling.connect(token); 
      
      // ICE/DTLS 建联
      await setupMediaTransport(signal, rootSpan); // 传递 span 上下文
      
      // 订阅远端流
      await subscribeRemoteTracks(signal, rootSpan);

      joinMeetingDuration.record(Date.now() - joinStart, { 'video.meeting.id': meetingId });
      rootSpan.setStatus({ code: SpanStatusCode.OK });
    } catch (e) {
      rootSpan.recordException(e);
      rootSpan.setStatus({ code: SpanStatusCode.ERROR, message: e.message });
      throw e;
    } finally {
      rootSpan.end();
    }
  });
}

// 2. 首帧渲染埋点 (在 VideoElement onloadeddata / onplaying 回调中)
export function onFirstFrameRendered(trackId: string, meetingId: string, delayMs: number) {
  const span = tracer.startSpan('Client.RenderFirstFrame', {
    attributes: {
      'video.track.id': trackId,
      'video.meeting.id': meetingId,
    },
  });
  span.setAttribute('video.client.first_frame_delay_ms', delayMs);
  firstFrameDelay.record(delayMs, { 'video.meeting.id': meetingId, 'video.track.id': trackId });
  span.end();
}

// 3. 定时上报 RTCStats (集成到 getStats 循环)
export function reportRtcStats(pc: RTCPeerConnection, meetingId: string) {
  pc.getStats().then(stats => {
    stats.forEach(report => {
      if (report.type === 'inbound-rtp' && report.kind === 'video') {
        const attrs = {
          'video.meeting.id': meetingId,
          'video.track.id': report.trackIdentifier,
          'video.track.codec': report.codecId ? stats.get(report.codecId)?.mimeType : 'unknown',
        };
        // 上报关键指标
        meter.createObservableGauge('video.client.inbound.bitrate_kbps', {
          callback: (observableResult) => {
            observableResult.observe(report.bytesReceived * 8 / 1000, attrs); // 简化计算
          }
        });
        // 丢包率、抖动、帧率等同理...
      }
    });
  });
}

groups:
- name: video-meeting-critical
  rules:
  # 入会成功率骤降
  - alert: MeetingJoinSuccessRateDrop
    expr: |
      sum(rate(video_meeting_join_total{result="success"}[5m])) 
      / 
      sum(rate(video_meeting_join_total[5m])) < 0.95
    for: 3m
    labels:
      severity: critical
      team: rtc-platform
    annotations:
      summary: "视频会议入会成功率低于 95%"
      description: "当前成功率: {{ $value | humanizePercentage }}, 请检查信令集群或网关日志。"

  # 客户端首帧渲染 P99 超过 5s
  - alert: ClientFirstFrameDelayHigh
    expr: |
      histogram_quantile(0.99, sum(rate(video_client_first_frame_delay_ms_bucket[5m])) by (le)) > 5000
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: "客户端首帧渲染延迟 P99 超过 5 秒"
      description: "可能由 ICE 失败、服务端分配慢或客户端解码器初始化异常导致。"

  # SFU 转发队列积压 (自定义指标)
  - alert: SFUForwardQueueBacklog
    expr: video_sfu_forward_queue_length > 1000
    for: 2m
    labels:
      severity: critical
    annotations:
      summary: "SFU 节点 {{ $labels.instance }} 转发队列积压"
      description: "当前队列长度 {{ $value }}，疑似 CPU 饱和或下游网络阻塞。"

graph LR
    subgraph Edge Node (媒体服务器节点)
        MS[Media Server] -->|Localhost:4317| SC[OTel Collector Sidecar]
        SC -->|Batch/Compress| GW[Collector Gateway Cluster]
    end
    subgraph Control Plane
        GW -->|Tail Sampling| Tempo[(Tempo/Jaeger)]
        GW -->|Metrics Transform| Prom[(Prometheus/VM)]
        GW -->|Log Route| Loki[(Loki/ES)]
    end

# otel-collector-config.yaml
receivers:
  otlp:
    protocols:
      grpc:
        max_recv_msg_size_mib: 64  # 媒体服务器批量上报体积大
      http:

processors:
  # 1. 资源保护：防止单实例 OOM 导致节点级雪崩
  memory_limiter:
    check_interval: 1s
    limit_mib: 1024
    spike_limit_mib: 256

  # 2. 核心：尾部采样 - 保留所有失败/慢调用/重点会议
  tail_sampling:
    decision_wait: 30s          # 等待 Span 组装完成
    num_traces: 50000           # 内存中缓存 Trace 数上限
    expected_new_traces_per_sec: 20000
    policies:
      - name: "rtc-critical-errors"
        type: string_attribute
        string_attribute:
          key: "error"
          values: ["true"]
      - name: "rtc-high-latency"
        type: latency
        latency:
          threshold_ms: 2000    # 入会/首帧 > 2s 全量保留
      - name: "rtc-vip-meetings"
        type: string_attribute
        string_attribute:
          key: "video.meeting.priority"
          values: ["P0", "VIP"] # 业务侧打标的重点会议
      - name: "rtc-probabilistic"
        type: probabilistic
        probabilistic:
          sampling_percentage: 1.0 # 其余 1% 采样

  # 3. 指标聚合降本 - 核心省钱手段
  metricstransform:
    transforms:
      - include: "video.sfu.forward.duration_ms"
        action: update
        operations:
          - action: aggregate
            aggregation: "sum,count,min,max" # 仅保留统计量，丢弃直方图桶细节
      - include: "video.client.*"
        action: update
        operations:
          - action: aggregate
            aggregation: "sum,count"

  # 4. 属性合规清洗 - 防止高基数属性入库
  attributes:
    actions:
      - key: "rtp.sequence_number"      # 严禁入库
        action: delete
      - key: "user.id"                  # 高基数 -> Hash 低基数
        action: hash
        hash_config:
          algorithm: sha256
          output_attribute: "user.id.hash"
      - key: "video.track.id"           # 高基数 -> 仅保留前缀/类型
        action: extract
        pattern: "^(video|audio|screen)_"
        target_key: "video.track.type"

  # 5. 批量压缩上报
  batch:
    timeout: 10s
    send_batch_max_size: 8192
    send_batch_size: 4096

exporters:
  # Tempo 支持原生 TraceQL，适合高基数检索
  otlp/tempo:
    endpoint: "tempo-gateway:4317"
    tls:
      insecure: true
  # VictoriaMetrics/Prometheus 存储聚合后指标
  prometheusremotewrite/vm:
    endpoint: "http://vm-insert:8480/api/v1/write"
    resource_to_telemetry_conversion:
      enabled: true

service:
  pipelines:
    traces:
      receivers: [otlp]
      processors: [memory_limiter, tail_sampling, attributes, batch]
      exporters: [otlp/tempo]
    metrics:
      receivers: [otlp]
      processors: [memory_limiter, metricstransform, batch]
      exporters: [prometheusremotewrite/vm]

graph TD
    A[RTC Engine 事件回调] --> B{采样决策器}
    B -- 1% 正常 / 100% 错误 --> C[本地 mmap/DB 缓冲区]
    C --> D[上传调度器]
    D -- WiFi+充电/强网 --> E[批量压缩 gRPC/HTTP/2]
    D -- 4G/弱网/低电量 --> F[本地持久化 延迟上传]
    E --> G[Collector Sidecar]
    F --> G

// Android 示例：WorkManager + NetworkCallback 实现智能调度
class TraceUploadWorker(context: Context, params: WorkerParameters) : CoroutineWorker(context, params) {
    override suspend fun doWork(): Result {
        val connectivity = ConnectivityManager.from(applicationContext)
        val network = connectivity.activeNetwork ?: return Result.retry()
        val capabilities = connectivity.getNetworkCapabilities(network)
        
        // 策略：非计费网络(WiFi/5G)且电量>20% -> 立即上传
        val isUnmetered = capabilities?.hasCapability(NET_CAPABILITY_NOT_METERED) == true
        val batteryLevel = getBatteryLevel()
        
        val batchSize = if (isUnmetered && batteryLevel > 20) 500 else 50 // 弱网/低电量减小批次
        val delayMs = if (isUnmetered) 0 else 30_000 // 弱网延迟30s合并
        
        traceRepository.flushBatch(batchSize, delayMs)
        return Result.success()
    }
}

# .github/workflows/otel-lint.yml
jobs:
  otel-lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Check Semantic Conventions
        run: |
          # 1. 校验属性命名规范 (video.meeting.* 前缀)
          go run github.com/open-telemetry/otel-lint/cmd/otel-lint@latest 
            --config .otel-lint.yaml ./...
          
          # 2. 自定义规则：禁止在 Span Attribute 记录高基数字段
          #    通过 AST 分析代码 (go/analysis, typescript-eslint)
          go run ./tools/otel-attr-checker ./...

// 禁止 span.SetAttributes(attribute.String("user.id", ...)) 
// 禁止 span.SetAttributes(attribute.Int("rtp.seq", ...))
func run(pass *analysis.Pass) (interface{}, error) {
    for _, file := range pass.Files {
        ast.Inspect(file, func(n ast.Node) bool {
            call, ok := n.(*ast.CallExpr)
            if !ok { return true }
            // 识别 tracer.Start 或 span.SetAttributes 调用
            if isOtelAttrCall(call) {
                for _, arg := range call.Args {
                    if kv, ok := arg.(*ast.CompositeLit); ok {
                        checkKeyValue(pass, kv)
                    }
                }
            }
            return true
        })
    }
    return nil, nil
}

# 压测后自动执行的对比脚本
#!/bin/bash
# 1. 从 VictoriaMetrics 拉取本次压测指标
curl -s "http://vm-select:8481/api/v1/query_range?query=histogram_quantile(0.99, rate(video_client_join_duration_ms_bucket[1m]))&start=$START&end=$END&step=10s" > current.json

# 2. 从基线存储拉取上版本基线
curl -s "http://baseline-store/baseline/video_client_join_duration_ms_p99.json" > baseline.json

# 3. Python 判定逻辑
python3 <<EOF
import json, sys
cur = [v[1] for v in json.load(open('current.json'))['data']['result'][0]['values']]
base = json.load(open('baseline.json'))['p99_ms']
avg_cur = sum(cur)/len(cur)
regression = (avg_cur - base) / base * 100

print(f"Current P99: {avg_cur:.0f}ms, Baseline: {base}ms, Regression: {regression:.1f}%")
if regression > 10: # 超过 10% 退化
    print("::error::Performance Regression Detected: Join Meeting P99 increased > 10%")
    sys.exit(1)
EOF

# 伪代码：基于 Trace 数据计算关键路径
def analyze_critical_path(trace_spans):
    # 1. 构建调用图
    graph = build_dag(trace_spans) # Node: Span, Edge: Parent-Child
    
    # 2. 识别业务关键节点
    key_nodes = {
        "join_entry": find_span(graph, name="Client.JoinMeeting"),
        "ice_done": find_span(graph, name="ICE.Connected"),
        "dtls_done": find_span(graph, name="DTLS.Handshake"),
        "first_frame": find_span(graph, name="Client.RenderFirstFrame"),
    }
    
    # 3. 计算阶段耗时
    stages = {
        "signaling_negotiation": key_nodes["ice_done"].start - key_nodes["join_entry"].start,
        "media_transport_setup": key_nodes["dtls_done"].end - key_nodes["ice_done"].start,
        "first_frame_decode_render": key_nodes["first_frame"].end - key_nodes["dtls_done"].end,
    }
    
    # 4. 异常判定：某阶段耗时 > P99 基线 * 1.5
    anomalies = {k: v for k, v in stages.items() if v > BASELINES[k] * 1.5}
    return anomalies, stages

def generate_rca_report(anomaly_context: dict) -> str:
    prompt = f"""
    角色：资深 RTC 架构师。
    任务：根据以下异常上下文，生成故障复盘报告（含现象、疑似根因、验证建议、修复方向）。
    
    异常现象：{anomaly_context['symptom']} (入会失败率从 0.1% 升至 12%)
    时间窗口：{anomaly_context['time_range']}
    关键关联特征（Top 3）：
    {json.dumps(anomaly_context['top_correlations'], indent=2)}
    
    典型错误 Trace 片段：
    {anomaly_context['sample_error_trace']}
    
    相关部署变更：
    {anomaly_context['recent_deployments']}
    
    请按：1. 故障定级 2. 影响范围 3. 根因假设(按可能性排序) 4. 验证步骤 5. 修复建议 格式输出。
    """
    return llm_client.chat(prompt)