# CPU饱和度：连续5分钟load1 > 核心数 * 0.8
node_load1 / count(node_cpu_seconds_total{mode="system"}) > 0.8

# 内存压力：可用内存 < 总量15%
(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.15

# 网卡丢包率：每秒丢包/总包 > 0.5%
rate(node_network_receive_errs_total[5m]) / rate(node_network_receive_packets_total[5m]) > 0.005

// 关键指标示例（Prometheus Go客户端）
var (
    // 并发会议/用户数
    ActiveConferences = prometheus.NewGaugeVec(prometheus.GaugeOpts{
        Name: "vc_active_conferences_total",
        Help: "当前活跃会议数",
    }, []string{"region", "cluster"})

    // 会议加入成功率
    JoinSuccessRate = prometheus.NewCounterVec(prometheus.CounterOpts{
        Name: "vc_join_attempts_total",
        Help: "会议加入尝试总数",
    }, []string{"result", "client_type"})

    // 端到端延迟（E2E Latency），单位ms
    E2ELatency = prometheus.NewHistogramVec(prometheus.HistogramOpts{
        Name:    "vc_e2e_latency_ms",
        Help:    "端到端媒体延迟",
        Buckets: []float64{50, 100, 150, 200, 300, 500, 1000},
    }, []string{"codec", "region"})

    // 丢包率（发送/接收端上报）
    PacketLossRate = prometheus.NewHistogramVec(prometheus.HistogramOpts{
        Name:    "vc_packet_loss_ratio",
        Help:    "媒体流丢包率",
        Buckets: []float64{0.001, 0.01, 0.05, 0.1, 0.2, 0.5},
    }, []string{"direction", "codec"})

    // 关键帧请求频率（反映网络抖动）
    FIRRate = prometheus.NewCounterVec(prometheus.CounterOpts{
        Name: "vc_fir_requests_total",
        Help: "FIR/NACK请求总数",
    }, []string{"stream_type"})
)

scrape_configs:
  # 媒体服务器集群（基于Consul/K8s ServiceDiscovery）
  - job_name: 'vc-media-server'
    consul_sd_configs:
      - server: 'consul.infra:8500'
        services: ['vc-media']
    relabel_configs:
      - source_labels: [__meta_consul_tags]
        regex: '.*enable_metrics=true.*'
        action: keep
      - source_labels: [__meta_consul_service_metadata_version]
        target_label: version

  # Blackbox网络探测
  - job_name: 'blackbox-icmp'
    metrics_path: /probe
    params:
      module: [icmp]
    static_configs:
      - targets:
          - 'media-cluster-1.vc.internal'
          - 'media-cluster-2.vc.internal'
          - 'turn.vc.internal'
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - target_label: __address__
        replacement: 'blackbox-exporter:9115'

  # 客户端聚合指标（Pushgateway模式，适合短生命周期上报）
  - job_name: 'vc-client-metrics'
    honor_timestamps: true
    static_configs:
      - targets: ['pushgateway.vc:9091']

groups:
- name: vc_recording_rules
  interval: 30s
  rules:
  # 区域级会议并发数
  - record: vc:active_conferences:region:sum
    expr: sum by (region) (vc_active_conferences_total)

  # 5分钟加入成功率
  - record: vc:join_success_rate:5m
    expr: |
      sum by (region, client_type) (
        rate(vc_join_attempts_total{result="success"}[5m])
      )
      /
      sum by (region, client_type) (
        rate(vc_join_attempts_total[5m])
      )

  # P95端到端延迟
  - record: vc:e2e_latency_p95:5m
    expr: |
      histogram_quantile(0.95,
        sum by (le, region, codec) (
          rate(vc_e2e_latency_ms_bucket[5m])
        )
      )

  # 丢包率P95
  - record: vc:packet_loss_p95:5m
    expr: |
      histogram_quantile(0.95,
        sum by (le, direction, codec) (
          rate(vc_packet_loss_ratio_bucket[5m])
        )
      )

# prometheus.yml 全局配置片段
global:
  scrape_interval: 15s
  evaluation_interval: 30s

storage:
  tsdb:
    retention.time: 15d          # 热数据保留
    retention.size: 500GB        # 磁盘上限
    wal-compression: true        # WAL压缩节省IO
    min-block-duration: 2h       # 块最小持久化间隔

# 最近5分钟成功率，阈值：≥98%绿、95-98%黄、<95%红
vc:join_success_rate:5m

# 按目标IP聚合的P99 RTT
histogram_quantile(0.99,
  sum by (le, target) (
    rate(probe_icmp_duration_seconds_bucket{job="blackbox-icmp"}[5m])
  )
)

# P50/P90/P99 延迟对比
histogram_quantile(0.50, sum by (le) (rate(vc_e2e_latency_ms_bucket[5m])))
histogram_quantile(0.90, sum by (le) (rate(vc_e2e_latency_ms_bucket[5m])))
histogram_quantile(0.99, sum by (le) (rate(vc_e2e_latency_ms_bucket[5m])))

// 变量定义示例（JSON Model片段）
{
  "name": "region",
  "type": "query",
  "datasource": "Prometheus",
  "query": "label_values(vc_active_conferences_total, region)",
  "multi": true,
  "includeAll": true,
  "allValue": ".*"
},
{
  "name": "cluster",
  "type": "query",
  "datasource": "Prometheus",
  "query": "label_values(vc_active_conferences_total{region=~"$region"}, cluster)",
  "multi": true,
  "refresh": 1
}

groups:
- name: vc_alerts_p0
  interval: 30s
  rules:
  # 全区会议加入失败率飙升
  - alert: VC_Join_Failure_Rate_High
    expr: |
      (
        sum by (region) (rate(vc_join_attempts_total{result="failed"}[5m]))
        /
        sum by (region) (rate(vc_join_attempts_total[5m]))
      ) > 0.2
    for: 3m
    labels:
      severity: p0
      team: vc-platform
    annotations:
      summary: "区域 {{ $labels.region }} 会议加入失败率超过 20%"
      description: "当前失败率: {{ $value | humanizePercentage }}，持续 3 分钟。请立即检查信令网关与媒体集群状态。"
      runbook_url: "https://wiki.internal/runbooks/vc-join-failure"

  # 媒体服务器心跳丢失（实例级）
  - alert: VC_Media_Server_Down
    expr: up{job="vc-media-server"} == 0
    for: 2m
    labels:
      severity: p0
      team: vc-media
    annotations:
      summary: "媒体服务器 {{ $labels.instance }} 心跳丢失"
      description: "实例 {{ $labels.instance }} (cluster={{ $labels.cluster }}) 已无指标上报 2 分钟。"

- name: vc_alerts_p1
  interval: 60s
  rules:
  # 单集群P99延迟超阈值
  - alert: VC_E2E_Latency_P99_High
    expr: |
      histogram_quantile(0.99,
        sum by (le, cluster) (rate(vc_e2e_latency_ms_bucket[5m]))
      ) > 500
    for: 10m
    labels:
      severity: p1
      team: vc-media
    annotations:
      summary: "集群 {{ $labels.cluster }} 端到端延迟 P99 超过 500ms"
      description: "当前 P99: {{ $value | printf "%.0f" }}ms，持续 10 分钟。疑似网络拥塞或媒体节点过载。"

  # 丢包率异常
  - alert: VC_Packet_Loss_High
    expr: |
      histogram_quantile(0.95,
        sum by (le, cluster, direction) (rate(vc_packet_loss_ratio_bucket[5m]))
      ) > 0.1
    for: 5m
    labels:
      severity: p1
      team: vc-network
    annotations:
      summary: "集群 {{ $labels.cluster }} {{ $labels.direction }} 向丢包率 P95 超过 10%"
      description: "建议排查转发链路、TURN服务器带宽及客户端网络环境。"

- name: vc_alerts_p2
  interval: 5m
  rules:
  # 资源饱和预警
  - alert: VC_Media_CPU_Saturation
    expr: |
      (sum by (instance) (rate(node_cpu_seconds_total{mode="system",job="vc-media-server"}[5m]))
       /
       sum by (instance) (rate(node_cpu_seconds_total{job="vc-media-server"}[5m]))) > 0.85
    for: 30m
    labels:
      severity: p2
      team: vc-media
    annotations:
      summary: "媒体节点 {{ $labels.instance }} CPU 系统态占比持续 > 85%"
      description: "建议评估扩容或流量调度策略。"

  # 证书过期预警（Blackbox HTTPS探测）
  - alert: VC_TLS_Cert_Expiring_Soon
    expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 14
    for: 1h
    labels:
      severity: p2
      team: vc-infra
    annotations:
      summary: "域名 {{ $labels.target }} TLS 证书将在 14 天内过期"
      description: "过期时间: {{ $value | humanizeDuration }}。请安排证书轮换。"

# alertmanager.yml 核心片段
route:
  group_by: ['alertname', 'region', 'cluster', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'default'
  routes:
    - match:
        severity: p0
      receiver: 'p0-oncall'
      continue: true
    - match:
        severity: p1
      receiver: 'p1-oncall'
      continue: true
    - match:
        severity: p2
      receiver: 'p2-ticket'
      continue: true

# 抑制规则：P0触发时抑制同集群P1/P2，避免告警风暴
inhibit_rules:
  - source_match:
      severity: p0
    target_match_re:
      severity: p1|p2
    equal: ['cluster', 'region']

receivers:
- name: 'default'
  email_configs:
    - to: 'vc-alerts@company.com'
      send_resolved: true

- name: 'p0-oncall'
  webhook_configs:
    - url: 'https://pagerduty.company.com/api/v1/alerts'
      http_config:
        basic_auth:
          username: 'prometheus'
          password: '${PAGERDUTY_KEY}'
  wechat_configs:
    - corp_id: '${WECHAT_CORP_ID}'
      api_secret: '${WECHAT_SECRET}'
      to_party: 'vc-oncall'

- name: 'p1-oncall'
  wechat_configs:
    - corp_id: '${WECHAT_CORP_ID}'
      api_secret: '${WECHAT_SECRET}'
      to_party: 'vc-oncall'

- name: 'p2-ticket'
  webhook_configs:
    - url: 'https://jira.company.com/rest/api/2/issue/'
      http_config:
        basic_auth:
          username: 'alertmanager'
          password: '${JIRA_TOKEN}'

{{ define "vc.wechat.card" }}
{
  "msgtype": "markdown",
  "markdown": {
    "content": "**【{{ .Labels.severity | upper }}】视频会议告警**n
    > **告警名称**: {{ .Annotations.summary }}n
    > **影响范围**: {{ .Labels.region }} / {{ .Labels.cluster }}n
    > **当前值**: {{ .Value }}n
    > **触发时间**: {{ .StartsAt.Format "2006-01-02 15:04:05" }}n
    > **详情**: {{ .Annotations.description }}n
    > **仪表盘**: [点击跳转]({{ .GeneratorURL }})n
    > **Runbook**: [处理手册]({{ .Annotations.runbook_url }})"
  }
}
{{ end }}

# mimir-values.yaml
mimir:
  structuredConfig:
    # 全局租户限制，防止单租户拖垮集群
    limits:
      ingestion_rate: 500000      # 每秒样本上限
      ingestion_burst_size: 1000000
      max_global_series_per_user: 5000000  # 核心：限制高基数
      max_global_series_per_metric: 10000  # 单指标标签组合上限
      retention_period: 13m       # 热数据 13 个月（含下采样）
      # 下采样规则：1h 后降为 5m 粒度，30d 后降为 1h 粒度
      downsampling:
        - interval: 5m
          range: 1h-30d
        - interval: 1h
          range: 30d-13m

    # 存储后端（兼容 S3 API，如 MinIO、AWS S3、阿里云 OSS）
    blocks_storage:
      backend: s3
      s3:
        endpoint: s3.internal.company.com
        bucket_name: mimir-blocks
        access_key: ${S3_AK}
        secret_key: ${S3_SK}
        insecure: true

    # 查询前端高可用
    query_frontend:
      max_outstanding_requests_per_tenant: 1000
      query_sharding:
        enabled: true
        max_sources: 10

    # 压缩器配置（关键：合并小块、下采样）
    compactor:
      compaction_interval: 15m
      retention_delete_delay: 2h
      downsampling_concurrency: 4

# 部署规模建议（生产环境）
# Query-Frontend: 3 副本, 4C8G
# Querier: 6 副本, 8C16G (可水平扩容)
# Ingester: 6 副本, 8C32G (内存决定 WAL 吞吐)
# Compactor: 2 副本, 8C16G
# Distributor: 4 副本, 4C8G
# Store-Gateway: 3 副本, 4C16G (本地缓存盘 ≥ 200GB NVMe)

# prometheus.yml (各边缘集群统一下发)
remote_write:
  - url: "https://mimir-gateway.company.com/api/v1/push"
    # 租户 ID 由网关根据 mTLS 或 Header 注入，客户端无需感知
    # 此处演示显式指定租户（需 Mimir 启用 auth_header: X-Scope-OrgID）
    headers:
      X-Scope-OrgID: "vc-prod-${REGION}"
    queue_config:
      capacity: 100000
      max_shards: 200
      min_shards: 50
      max_samples_per_send: 5000
      batch_send_deadline: 5s
    # 关键：写入重试与熔断，防止网络抖动导致本地 WAL 堆积
    retry_on_rate_limit: true
    send_exemplars: true  # 关联 TraceID，打通指标-链路

# 查询全网所有租户的会议并发总和（需 Mimir 管理员权限或跨租户查询配置）
sum by (region) (vc_active_conferences_total{__tenant__=~"vc-prod-.*"})

graph LR
    A[WebRTC Client<br/>getStats()] -->|1. 定时聚合<br/>(10s/次)| B[Client SDK<br/>本地缓冲+批量压缩]
    B -->|2. HTTPS/QUIC<br/>(gRPC/Protobuf)| C[接入网关<br/>Nginx/Envoy]
    C -->|3. 解析/采样/脱敏| D[流式处理<br/>Flink/KSQL/Fluent Bit]
    D -->|4. 聚合为 Histogram| E[Pushgateway<br/>(按 region/cluster/version 分组)]
    E -->|5. Prometheus Scrape| F[Prometheus/Mimir]

// Web/React Native/Electron 通用采集器伪代码
class QoECollector {
  private pc: RTCPeerConnection;
  private buffer: MetricPoint[] = [];
  private readonly REPORT_INTERVAL = 10000; // 10s 上报一次
  private readonly MAX_BUFFER = 50;         // 离线缓存上限

  start() {
    setInterval(() => this.collectAndFlush(), this.REPORT_INTERVAL);
    // 页面卸载前强制上报
    window.addEventListener('beforeunload', () => this.flush(true));
  }

  private async collectAndFlush() {
    if (!this.pc) return;
    const stats = await this.pc.getStats();
    const metrics = this.transform(stats); // 关键：转为 Prometheus 友好格式
    this.buffer.push(...metrics);
    if (this.buffer.length >= 20) this.flush();
  }

  private transform(report: RTCStatsReport): MetricPoint[] {
    const points: MetricPoint[] = [];
    const now = Date.now();
    const labels = {
      sdk_version: SDK_VERSION,
      client_type: 'web', // ios/android/electron
      region: getUserRegion(), // 由后端下发或 IP 归属
      isp: await getISP(),     // 可选，延迟获取
      codec: this.getCurrentCodec(report)
    };

    report.forEach((stat) => {
      if (stat.type === 'inbound-rtp' && stat.kind === 'video') {
        // 核心指标：丢包率、抖动、帧率、分辨率、解码耗时
        points.push(this.makeHistogram('vc_client_inbound_packet_loss_ratio', stat.packetsLost / (stat.packetsLost + stat.packetsReceived), labels));
        points.push(this.makeHistogram('vc_client_inbound_jitter_ms', stat.jitter * 1000, labels));
        points.push(this.makeGauge('vc_client_inbound_frame_rate', stat.framesPerSecond, labels));
        points.push(this.makeGauge('vc_client_inbound_frame_width', stat.frameWidth, labels));
        points.push(this.makeHistogram('vc_client_decode_time_ms', stat.totalDecodeTime / stat.framesDecoded * 1000, labels));
      }
      if (stat.type === 'outbound-rtp') {
        points.push(this.makeHistogram('vc_client_outbound_packet_loss_ratio', stat.packetsLost / stat.packetsSent, labels));
        points.push(this.makeGauge('vc_client_outbound_bitrate_kbps', stat.bytesSent * 8 / 1000, labels));
        points.push(this.makeHistogram('vc_client_rtt_ms', stat.roundTripTime * 1000, labels));
      }
      if (stat.type === 'candidate-pair' && stat.state === 'succeeded') {
        // 关键：连接类型、协议、候选对类型
        points.push(this.makeGauge('vc_client_connection_type', 1, { ...labels, type: stat.nominated ? 'nominated' : 'backup', protocol: stat.transport }));
      }
    });
    return points;
  }

  private flush(sync = false) {
    if (this.buffer.length === 0) return;
    const payload = this.encodeProtobuf(this.buffer); // Protobuf 压缩体积 ~ JSON 1/10
    this.buffer = [];
    navigator.sendBeacon('/api/v1/qoe/report', payload); // 可靠上报，不阻塞主线程
  }
}

-- Flink SQL: 从 Kafka 消费原始上报，聚合为 Prometheus Histogram 格式推送 Pushgateway
CREATE TABLE raw_qoe (
  sdk_version STRING,
  client_type STRING,
  region STRING,
  isp STRING,
  codec STRING,
  metric_name STRING,
  metric_value DOUBLE,
  event_time TIMESTAMP(3),
  WATERMARK FOR event_time AS event_time - INTERVAL '10' SECOND
) WITH ('connector'='kafka', 'topic'='client_qoe_raw', ...);

CREATE TABLE pushgateway_sink (
  job_name STRING,
  grouping_key MAP<STRING, STRING>, -- {instance="web-shanghai-chrome-h264"}
  metrics MAP<STRING, BYTES>        -- Prometheus Text Format 或 Protobuf
) WITH ('connector'='pushgateway', 'url'='http://pushgateway:9091');

-- 核心聚合逻辑：每 30 秒按维度计算 Histogram Buckets
INSERT INTO pushgateway_sink
SELECT
  'vc_client_qoe' as job_name,
  MAP[
    'region', region,
    'client_type', client_type,
    'sdk_version', sdk_version,
    'codec', codec,
    'isp', isp
  ] as grouping_key,
  -- 使用 HISTOGRAM 近似算法 (如 T-Digest 或固定 Bucket) 生成 bucket 计数
  -- 此处简化为生成 Text Format 字节流
  GENERATE_PROM_HISTOGRAM(metric_name, metric_value, ARRAY[0.001, 0.01, 0.05, 0.1, 0.2, 0.5]) as metrics
FROM raw_qoe
GROUP BY
  TUMBLE(event_time, INTERVAL '30' SECOND),
  region, client_type, sdk_version, codec, isp, metric_name;

# docker-compose.yml 片段
services:
  vmanomaly:
    image: victoriametrics/anomaly-detector:v1.9.0
    command:
      - --prometheus.url=http://mimir-querier:9090
      - --prometheus.auth.config=/etc/vmanomaly/auth.yaml
      - --model.dir=/data/models
      - --rule.config=/etc/vmanomaly/rules.yaml
      - --web.listen=:8080
    volumes:
      - ./vmanomaly/rules.yaml:/etc/vmanomaly/rules.yaml
      - ./vmanomaly/auth.yaml:/etc/vmanomaly/auth.yaml
      - vmanomaly_data:/data/models

# rules.yaml: 仅对核心指标开启异常检测
groups:
- name: vc_anomaly_detection
  interval: 1m
  rules:
  # 会议加入成功率：训练周期 7 天，季节性 1 天
  - expr: 'vc:join_success_rate:5m{region=~".+"}'
    model:
      type: "prophet"  # 或 "arima", "lstm" (需 GPU)
      seasonality: "1d"
      training_window: "7d"
      sensitivity: 3.0  # 灵敏度，越大越不易报警
    alert:
      name: "VC_Join_Success_Rate_Anomaly"
      severity: "p1"
      annotations:
        summary: "区域 {{ $labels.region }} 加入成功率出现异常波动"
        description: "当前值 {{ $value }} 偏离预测基线 {{ $predicted }}，异常得分 {{ $score }}。"

  # 端到端延迟 P99：双季节性（日/周）
  - expr: 'vc:e2e_latency_p99:5m{cluster=~".+"}'
    model:
      type: "prophet"
      seasonality: ["1d", "7d"]
      training_window: "14d"
      sensitivity: 2.5
    alert:
      name: "VC_E2E_Latency_Anomaly"
      severity: "p1"

# Alertmanager 接收 vmanomaly 的 Webhook
receivers:
- name: 'anomaly-webhook'
  webhook_configs:
    - url: 'http://alertmanager:9093/api/v2/alerts'
      # vmanomaly 发送格式需适配 Alertmanager 标准格式
      # 或在 vmanomaly 配置 alertmanager.url 直接推送

# .gitlab-ci.yml / GitHub Actions 片段
stages:
  - validate
  - inject
  - observe
  - verify
  - cleanup
  - report

chaos_inject:
  stage: inject
  script:
    - |
      # 使用 Chaos Mesh / LitmusChaos / 自研 Agent 注入故障
      kubectl apply -f chaos/network-loss-media-server.yaml
      # 记录注入时间戳到 Redis/Etcd 供后续校验
      redis-cli SET "chaos:start_ts" "$(date +%s)"

chaos_verify:
  stage: verify
  needs: [chaos_inject]
  script:
    - |
      # 等待监控系统反应窗口 (5min)
      sleep 300
      # 1. 校验 Alertmanager 是否收到预期告警
      ALERTS=$(curl -s "http://alertmanager:9093/api/v2/alerts?filter=alertname=VC_Packet_Loss_High")
      if echo "$ALERTS" | jq -e 'length > 0' > /dev/null; then
        echo "✅ 告警触发验证通过"
      else
        echo "❌ 告警未触发"; exit 1
      fi
      # 2. 校验 Grafana 面板数据是否反映故障 (通过 API 查询)
      VALUE=$(curl -s -G "http://mimir:9090/api/v1/query" --data-urlencode 'query=vc:packet_loss_p95:5m' | jq '.data.result[0].value[1]')
      if (( $(echo "$VALUE > 0.05" | bc -l) )); then
        echo "✅ 指标采集验证通过: $VALUE"
      else
        echo "❌ 指标未反映故障"; exit 1
      fi

chaos_cleanup:
  stage: cleanup
  when: always
  script:
    - kubectl delete -f chaos/network-loss-media-server.yaml --ignore-not-found
    - redis-cli DEL "chaos:start_ts"

# 混沌演练复盘报告: CHAOS-VC-01-2024Q3

**执行时间**: 2024-09-15 10:00 - 10:30
**执行人**: SRE Team
**故障类型**: 媒体服务器网络丢包 10%

## 验证结果
| 验证项 | 预期 | 实际 | 结论 |
|--------|------|------|------|
| 告警触发时效 | < 2 min | 1 min 45 sec | ✅ 通过 |
| 告警分级准确性 | P1 | P1 | ✅ 通过 |
| 告警风暴抑制 | 单条告警 | 单条告警 (抑制了 3 条 P2 资源告警) | ✅ 通过 |
| 仪表盘定位能力 | 实例级 | 实例级 (通过 `instance` 标签直达) | ✅ 通过 |
| Runbook 跳转有效性 | 可打开 | 可打开，步骤清晰 | ✅ 通过 |

## 发现问题 & 改进项
1. **问题**: 客户端感知指标 `vc_client_inbound_jitter_ms` 延迟 3 分钟才反映丢包。
   **根因**: 客户端上报周期 10s + Flink 窗口 30s + Pushgateway 推送 15s + Prometheus 抓取 15s = ~70s，叠加网络抖动。
   **改进**: 核心链路客户端改为 **WebSocket 长连接推送**，端到端延迟目标 < 15s。

2. **问题**: 告警通知中缺少“受影响会议估算数”。
   **改进**: Alertmanager 模板增加 `{{ $labels.region }}` 当前并发数查询逻辑（需 Alertmanager 0.27+ 模板函数支持或外挂 Enrichment 服务）。

#!/bin/bash
# find_high_cardinality.sh: 扫描 Prometheus/Mimir 当前最高基数指标 Top 20
PROM_URL="http://mimir-querier:9090"
TENANT="vc-prod-shanghai"

curl -s -G "${PROM_URL}/api/v1/label/__name__/values" 
  -H "X-Scope-OrgID: ${TENANT}" | jq -r '.data[]' | while read metric; do
  # 仅采样近 1 小时，避免全量扫描压垮查询器
  cardinality=$(curl -s -G "${PROM_URL}/api/v1/query" 
    -H "X-Scope-OrgID: ${TENANT}" 
    --data-urlencode "query=count by (__name__) (count by (__name__, ${metric}) (${metric}[1h]))" 
    | jq '.data.result[0].value[1] // "0"')
  echo "${cardinality} ${metric}"
done | sort -rn | head -20

# prometheus.yml scrape_configs 片段
- job_name: 'vc-media-server'
  # ...
  metric_relabel_configs:
    # 1. 直接丢弃高基数标签
    - source_labels: [__name__]
      regex: 'vc_.*_bucket'
      action: labeldrop
      regex: 'le'  # 保留 le 标签用于 Histogram，但丢弃其他
    - action: labeldrop
      regex: '(meeting_id|user_id|ssrc|track_id|participant_id)'

    # 2. 将高基数标签哈希为低基数桶（保留分布特征）
    - source_labels: [meeting_id]
      target_label: meeting_id_bucket
      modulus: 1000
      action: hashmod
    - action: labeldrop
      regex: 'meeting_id' # 丢弃原始 ID

# rules/cardinality_reduction.yml
groups:
- name: vc_cardinality_reduction
  interval: 1m
  rules:
  # 将会议级指标聚合为集群级，彻底消除 meeting_id
  - record: vc:cluster:media_bitrate_sum:1m
    expr: |
      sum by (cluster, codec, region) (
        vc_media_bitrate_total{job="vc-media-server"}
      )
  # 仅保留 Top 100 会议的详细指标（通过 TopK 过滤）
  - record: vc:top100_meetings:bitrate:1m
    expr: |
      topk by (meeting_id) (100, vc_media_bitrate_total{job="vc-media-server"})

# mimir structuredConfig limits 补充
limits:
  # 硬性拦截：单指标标签组合超过 1万 直接拒绝写入，返回 429
  max_global_series_per_metric: 10000
  # 软性告警：触发 Mimir 内部告警，推送至 Alertmanager
  cardinality_limit_metric_name: "mimir_series_per_metric_limit_exceeded"

# prometheus/alerts/release.yml: 发布期间临时加载的规则组（通过 Reloader 热加载）
groups:
- name: vc_release_guard
  interval: 15s  # 发布期高频评估
  rules:
  # 灰度版本错误率对比（金丝雀 vs 基线）
  - alert: VC_Release_Error_Rate_Higher_Than_Baseline
    expr: |
      (
        sum by (version) (rate(vc_join_attempts_total{result="failed", environment="canary"}[2m]))
        /
        sum by (version) (rate(vc_join_attempts_total{environment="canary"}[2m]))
      )
      >
      1.5 * (
        sum by (version) (rate(vc_join_attempts_total{result="failed", environment="stable"}[10m]))
        /
        sum by (version) (rate(vc_join_attempts_total{environment="stable"}[10m]))
      )
    for: 3m
    labels:
      severity: p0
      context: release
    annotations:
      summary: "灰度版本 {{ $labels.version }} 加入失败率显著高于稳定版基线"
      runbook_url: "https://wiki.internal/runbooks/release-rollback"

  # 关键指标波动率（方差系数 CV）
  - alert: VC_Release_Metric_Volatility_High
    expr: |
      (
        stddev_over_time(vc:e2e_latency_p95:1m{environment="canary"}[10m])
        /
        avg_over_time(vc:e2e_latency_p95:1m{environment="canary"}[10m])
      ) > 0.3
    for: 5m
    labels:
      severity: p1
      context: release

# .gitlab-ci.yml
stages:
  - build
  - deploy_canary
  - monitor_canary
  - promote_or_rollback
  - deploy_stable

monitor_canary:
  stage: monitor_canary
  image: curlimages/curl:latest
  timeout: 30m
  script:
    - |
      echo "等待灰度流量稳定..."
      sleep 120
      START_TIME=$(date +%s)
      while [ $(($(date +%s) - START_TIME)) -lt 1800 ]; do # 监控 30 分钟
        # 查询 Alertmanager 是否有 context=release 的活跃告警
        ACTIVE_ALERTS=$(curl -s "http://alertmanager:9093/api/v2/alerts?filter=context=release&silenced=false&inhibited=false" | jq 'length')
        if [ "$ACTIVE_ALERTS" -gt 0 ]; then
          echo "🚨 发现发布期告警，触发自动回滚"
          curl -X POST "https://deploy.company.com/api/rollback/${CI_PIPELINE_ID}"
          exit 1
        fi
        # 检查核心 SLO 是否达标 (成功率 > 99.5%)
        SLO=$(curl -s -G "http://mimir:9090/api/v1/query" --data-urlencode 'query=vc:join_success_rate:5m{environment="canary"}' | jq -r '.data.result[0].value[1]')
        if (( $(echo "$SLO < 0.995" | bc -l) )); then
          echo "🚨 SLO 未达标 ($SLO)，触发自动回滚"
          curl -X POST "https://deploy.company.com/api/rollback/${CI_PIPELINE_ID}"
          exit 1
        fi
        echo "✅ 监控正常，SLO: $SLO，等待下一轮..."
        sleep 60
      done
      echo "✅ 灰度观测期通过，准备全量发布"
  rules:
    - if: $CI_COMMIT_BRANCH == "main" || $CI_COMMIT_TAG

# 面板：加入成功率对比
vc:join_success_rate:5m{version=~"$VERSION_A|$VERSION_B"}

# 面板：延迟分位数对比 (Heatmap)
histogram_quantile(0.99, sum by (le, version) (rate(vc_e2e_latency_ms_bucket{version=~"$VERSION_A|$VERSION_B"}[5m])))

# 1. 会议加入成功率 (5分钟窗口)
sum by (region, version) (rate(vc_join_attempts_total{result="success"}[5m]))
/
sum by (region, version) (rate(vc_join_attempts_total[5m]))

# 2. 端到端延迟 P50/P90/P99 (利用 Recording Rule 加速)
vc:e2e_latency_p50:5m
vc:e2e_latency_p90:5m
vc:e2e_latency_p99:5m

# 3. 丢包率 P95 (按方向区分)
vc:packet_loss_p95:5m{direction="inbound"}
vc:packet_loss_p95:5m{direction="outbound"}

# 4. 并发会议数 & 并发用户数
sum by (region) (vc_active_conferences_total)
sum by (region) (vc_active_users_total)

# 5. 服务器资源饱和度 (USE)
# CPU: 系统态 > 60% 持续 15min
avg by (instance) (rate(node_cpu_seconds_total{mode="system"}[5m])) > 0.6
# 内存: 可用 < 20%
(node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) < 0.2
# 网卡: 吞吐量 > 80% 带宽上限 (需结合 node_network_speed_bytes 或固定阈值)
rate(node_network_transmit_bytes_total[5m]) > 0.8 * 1.25e9  # 10Gbps 网卡示例

# 6. 客户端版本分布 (热力图)
count by (sdk_version, client_type) (vc_client_session_active)

# 7. 关键帧请求率 (反映网络抖动)
rate(vc_fir_requests_total[5m]) / rate(vc_key_frames_encoded_total[5m])

# 8. TURN 服务器中继流量占比 (衡量直连率)
sum(rate(vc_turn_relay_bytes_total[5m])) / sum(rate(vc_total_media_bytes_total[5m]))

# 9. 证书过期天数
(probe_ssl_earliest_cert_expiry - time()) / 86400

# 10. 采集健康度
up{job="vc-media-server"} == 0
rate(prometheus_target_scrapes_exceeded_sample_limit_total[5m]) > 0