flowchart LR
    A[视频会议业务组件] -->|HTTP /metrics<br>gRPC /stats| B(Prometheus Agent /<br>OpenTelemetry Collector)
    C[基础设施节点] -->|Node Exporter<br>cAdvisor<br>DCGM Exporter| B
    D[网络设备/负载均衡] -->|SNMP Exporter| B
    B -->|remote_write| E[(Prometheus<br>Thanos/VictoriaMetrics)]
    E -->|Query API| F[Grafana Dashboard]
    E -->|Alert Rule| G[Alertmanager]
    G -->|Webhook| H[企业微信/钉钉/OnCall]

# values-victoriametrics.yaml
victoria-metrics-single:
  resources:
    limits:
      memory: 16Gi
      cpu: "8"
    requests:
      memory: 8Gi
      cpu: "4"
  persistence:
    size: 500Gi
    storageClass: ssd
  retentionPeriod: "3m"          # 热数据保留 3 个月
  retentionStorage: "500Gi"
  extraArgs:
    - "--search.maxPointsPerTimeseries=30000"
    - "--maxConcurrentQueries=64"

// main.go 片段
import (
    "github.com/prometheus/client_golang/prometheus/promhttp"
    "net/http"
)

func initMetrics() {
    http.Handle("/metrics", promhttp.Handler())
    go http.ListenAndServe(":9090", nil)
}

// 关键业务指标定义
var (
    conferenceActive = prometheus.NewGaugeVec(
        prometheus.GaugeOpts{Name: "sfu_conference_active_total", Help: "当前活跃会议数"},
        []string{"cluster", "node"},
    )
    peerConnected = prometheus.NewCounterVec(
        prometheus.CounterOpts{Name: "sfu_peer_connected_total", Help: "累计入会人数"},
        []string{"cluster", "node", "result"}, // result: success/failed
    )
    iceConnectDuration = prometheus.NewHistogramVec(
        prometheus.HistogramOpts{
            Name:    "sfu_ice_connect_duration_seconds",
            Help:    "ICE 连接建立耗时",
            Buckets: []float64{0.1, 0.3, 0.5, 1, 2, 5, 10},
        },
        []string{"cluster", "node", "network_type"}, // wifi/4g/5g/ethernet
    )
)

apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: sfu-monitor
  labels:
    release: prometheus
spec:
  selector:
    matchLabels:
      app: sfu-server
  endpoints:
  - port: metrics
    interval: 15s
    path: /metrics
    scheme: http
  namespaceSelector:
    matchNames: ["video-prod"]

# 会议创建成功率（5分钟窗口）
sum(rate(sfu_conference_create_total{result="success"}[5m])) by (cluster)
/
sum(rate(sfu_conference_create_total[5m])) by (cluster)

# 端到端 P95 延迟（需客户端上报或服务端采样）
histogram_quantile(0.95,
  sum(rate(sfu_rtt_seconds_bucket[5m])) by (le, cluster)
)

# 节点带宽利用率告警阈值判定
(max by (node) (rate(node_network_transmit_bytes_total[1m])) * 8)
/
node_network_speed_bytes{device=~"eth.*"} * 100 > 80

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: video-conference-alerts
spec:
  groups:
  - name: sfu-availability
    rules:
    - alert: SFUNodeDown
      expr: up{job="sfu-server"} == 0
      for: 2m
      labels:
        severity: critical
        team: video-infra
      annotations:
        summary: "SFU 节点 {{ $labels.node }} 心跳丢失"
        description: "节点 {{ $labels.node }} ({{ $labels.instance }}) 超过 2 分钟无心跳，请检查容器状态或宿主机网络。"
        runbook_url: "https://wiki.example.com/runbook/sfu-node-down"

    - alert: SFUHighICEFailureRate
      expr: |
        sum(rate(sfu_ice_connect_total{result="failed"}[5m])) by (cluster, node)
        /
        sum(rate(sfu_ice_connect_total[5m])) by (cluster, node) > 0.05
      for: 5m
      labels:
        severity: warning
        team: video-infra
      annotations:
        summary: "节点 {{ $labels.node }} ICE 连接失败率超过 5%"
        description: "当前失败率 {{ $value | humanizePercentage }}，疑似网络抖动或 TURN 服务异常。"

  - name: capacity-planning
    rules:
    - alert: DiskSpaceRunningLow
      expr: (node_filesystem_avail_bytes{mountpoint="/data"} / node_filesystem_size_bytes{mountpoint="/data"}) < 0.15
      for: 30m
      labels:
        severity: info
        team: video-infra
      annotations:
        summary: "节点 {{ $labels.instance }} 数据盘剩余空间不足 15%"

# otel-collector-config.yaml 片段
processors:
  # 1. 降低基数：移除 session_id, user_id, track_id 等高基数标签
  attributes/filter_high_cardinality:
    actions:
      - key: "session_id"
        action: "delete"
      - key: "user_id"
        action: "delete"
      - key: "track_id"
        action: "delete"

  # 2. 客户端侧采样：仅上报 10% 会话的全量 stats，其余仅上报 KQI 汇总
  probabilistic_sampler:
    sampling_percentage: 10
    hash_seed: "webrtc_qos"

  # 3. 业务语义化转换：将原始 stats 转为 KQI Gauge/Histogram
  # 需开发自定义 Processor (Go) 或使用 k8sattributes + metricstransform 组合
  # 示例：计算 MOS 并输出为 gauge
  metricstransform/mos_calc:
    transforms:
      - include: "webrtc_inbound_jitter"
        action: "update"
        operations:
          - action: "add_label"
            new_label: "metric_type"
            new_value: "kqi_mos"
          # 此处需配合自定义逻辑计算 R 值，建议在 Exporter 侧完成

exporters:
  prometheusremotewrite:
    endpoint: "http://victoriametrics-vminsert:8480/insert/0/prometheus/api/v1/write"
    tls:
      insecure: true
    # 关键：开启批量压缩，减少网络开销
    compression: "snappy"

flowchart TB
    subgraph Edge["边缘/区域集群 (10+)"]
        direction TB
        Agent[(Prometheus Agent Mode)] --> |Remote Write| RegionalVM[(VictoriaMetrics Single/Cluster)]
        NodeExp[Node Exporter] --> Agent
        SFUExp[SFU Metrics] --> Agent
    end

    subgraph Central["中央管控平台"]
        direction TB
        GlobalVM[(VictoriaMetrics Cluster / Thanos Query)]
        AlertMgr[Alertmanager HA]
        Grafana[Grafana Enterprise / Multi-tenant]
    end

    RegionalVM --> |Remote Write / Replication| GlobalVM
    GlobalVM --> Grafana
    GlobalVM --> AlertMgr

# Prometheus/VM Agent 侧 scrape_config 片段
scrape_configs:
  - job_name: 'sfu-hybrid'
    # 1. K8s 内部服务发现
    kubernetes_sd_configs:
      - role: pod
        namespaces:
          names: ['video-prod', 'video-staging']
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_label_app]
        regex: 'sfu-server'
        action: keep
      # 2. 融合 CMDB 资产标签 (通过外部 HTTP API 获取)
      - source_labels: [__meta_kubernetes_pod_node_name]
        target_label: __param_hostname
      - source_labels: [__param_hostname]
        target_label: instance
      - source_labels: [__param_hostname]
        url: http://cmdb-api.internal/api/v1/host/labels
        regex: '(.+)'
        replacement: '${1}'
        action: labelmap
        # CMDB 返回 JSON: {"dc": "bj-01", "rack": "A03", "owner": "video-team"}

// Dashboard JSON 片段：单会议诊断面板
{
  "type": "table",
  "datasource": "VictoriaMetrics",
  "targets": [{ "expr": "sfu_conference_active{cluster=~"$cluster"}", "format": "table" }],
  "fieldConfig": {
    "overrides": [
      {
        "matcher": { "id": "byName", "options": "meeting_id" },
        "properties": [
          { "id": "links", "value": [
            { "title": "🔍 链路追踪", "url": "http://tempo:3200/tempo/traces/${__cell.value}", "targetBlank": true },
            { "title": "📜 会议日志", "url": "http://loki:3100/loki?query={app="sfu", meeting_id="${__cell.value}"}", "targetBlank": true },
            { "title": "📊 质量趋势", "url": "/d/single-meeting-qos?var-meeting_id=${__cell.value}", "targetBlank": false }
          ]}
        ]
      }
    ]
  }
}

# 禁止敏感标签入库
relabel_configs:
  - source_labels: [__meta_kubernetes_pod_label_user_id]
    action: labeldrop  # 直接丢弃
  - source_labels: [__meta_kubernetes_pod_label_meeting_id]
    action: replace
    target_label: meeting_id_hash
    regex: '(.+)'
    replacement: '${1}' # 配合 hashmod 或外部哈希处理
    # 建议在 OTel Collector 侧用 `attributes/hasher` 处理

# Loki Promtail pipeline_stages
pipeline_stages:
  - json:
      expressions:
        user_id: user_id
        ip: client_ip
        message: log
  - labels:
      user_id_hash: # 不入标签，仅用于后续处理
  - replace:
      expression: '(\d{3})\d{4}(\d{4})'
      replace: '$1****$2'
      source: user_id # 假设 user_id 是手机号
  - replace:
      expression: '(\d+\.\d+\.\d+)\.d+'
      replace: '$1.*'
      source: ip
  - output:
      source: message # 处理后的 message 入库

# 1. 当前活跃序列数 Top 10 Job
topk(10, sum by (job) (count({__name__=~".+"})))

# 2. 日志写入量趋势
sum(rate(loki_distributor_bytes_received_total[5m])) by (namespace)

# 3. 存储增长率预测 (未来 30 天)
predict_linear(vm_data_size_bytes[7d], 30*24*3600)

# 4. 单租户/业务线成本分摊 (需配合 VictoriaMetrics 租户统计)
sum by (tenant_id) (vm_tenant_storage_bytes)

# Runbook: SFUHighICEFailureRate

## 1. 告警含义
ICE 连接建立失败率 > 5%，用户无法入会或入会即掉线。

## 2. 影响范围判断
- 查看 Grafana: `SFU Cluster Overview` -> 确认是单节点、单机房还是全网。
- 检查关联告警：是否伴随 `TURNServerDown`、`NetworkPacketLoss`。

## 3. 定界步骤 (SOP)
1. [ ] 点击告警面板链接跳转 **单节点诊断 Dashboard**。
2. [ ] 查看 `ICE 状态分布` 面板：`failed` / `timeout` / `cancelled` 占比。
3. [ ] 跳转 Loki 查询该节点 `sfu` 日志：`{app="sfu", node="xxx"} |= "ICE" | json | level="error"`。
4. [ ] 检查 TURN 服务状态：`curl -s http://turn-internal:8080/health`。
5. [ ] 检查网络链路：`mtr -r -c 10 <客户端IP>` (需客服协助获取)。

## 4. 常见根因 & 处置
| 根因现象 | 日志特征 | 处置动作 | 验证方式 |
| :--- | :--- | :--- | :--- |
| TURN 服务挂了 | `TURN allocation failed` | 重启 TURN Pod / 扩容 | ICE 成功率恢复 > 99% |
| 客户端网络严重 NAT | `candidate type: relay` 占比高但失败 | 引导用户切换网络 / 检查 TURN 带宽 | 单用户测试通过 |
| SFU 公网 IP 耗尽 | `no available ip for candidate` | 扩容 EIP 池 / 释放僵尸 IP | 新会议分配到 IP 正常 |

## 5. 升级路径
- 单节点 > 10min 未恢复 -> 升级至 Team Leader
- 单机房 > 5% 会议受影响 -> 触发 **跨机房流量切换预案** -> 升级至 值班经理

## 6. 事后复盘模板链接
[Postmortem Template](https://wiki.example.com/postmortem/template)