# prometheus.yml 片段
scrape_configs:
  - job_name: 'media-server-nodes'
    static_configs:
      - targets: ['media1.example.com:9100', 'media2.example.com:9100']
        labels:
          cluster: 'prod-east'
          role: 'sfu'

  - job_name: 'blackbox-icmp'
    metrics_path: /probe
    params:
      module: [icmp]
    static_configs:
      - targets: ['media1.example.com', 'media2.example.com', 'turn.example.com']
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-exporter:9115

- job_name: 'k8s-pods-media'
  kubernetes_sd_configs:
    - role: pod
      namespaces:
        names: ['video-conferencing']
  relabel_configs:
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
      action: keep
      regex: true
    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
      action: replace
      target_label: __metrics_path__
      regex: (.+)
    - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
      action: replace
      regex: ([^:]+)(?::d+)?;(d+)
      replacement: $1:$2
      target_label: __address__
    - action: labelmap
      regex: __meta_kubernetes_pod_label_(.+)

# alerts-webrtc.yml
groups:
  - name: webrtc-qos-critical
    interval: 30s
    rules:
      # P0: 单会议丢包率持续超阈值
      - alert: WebRTCHighPacketLoss
        expr: |
          (
            sum by (conference_id) (rate(webrtc_inbound_rtp_packets_lost_total[2m]))
            /
            sum by (conference_id) (rate(webrtc_inbound_rtp_packets_total[2m]))
          ) > 0.1
        for: 3m
        labels:
          severity: critical
          team: video-infra
        annotations:
          summary: "会议 {{ $labels.conference_id }} 入向丢包率超过 10%"
          description: "持续 3 分钟丢包率 > 10%，可能影响音视频质量，建议检查媒体节点 {{ $labels.instance }} 网络/负载。"
          runbook_url: "https://wiki.example.com/runbook/webrtc-packet-loss"

      # P1: RTT 持续偏高
      - alert: WebRTCHighRTT
        expr: |
          histogram_quantile(0.95, sum by (le, conference_id) (rate(webrtc_round_trip_time_seconds_bucket[5m]))) > 0.4
        for: 5m
        labels:
          severity: warning
          team: video-infra
        annotations:
          summary: "会议 {{ $labels.conference_id }} P95 RTT 超过 400ms"
          description: "跨区/弱网环境下常见，建议核对用户接入节点就近调度策略。"

      # P2: 可用带宽低于编码码率
      - alert: WebRTCBandwidthInsufficient
        expr: |
          (webrtc_available_outgoing_bitrate / webrtc_outbound_rtp_bytes_total * 8) < 0.8
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "可用带宽不足以支撑当前编码码率"
          description: "建议触发降级策略：降低分辨率/帧率或启用 FEC/NACK。"

# alertmanager.yml 关键片段
route:
  group_by: ['alertname', 'cluster', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'default'
  routes:
    - match:
        severity: critical
      receiver: 'pagerduty-critical'
      continue: true
    - match:
        team: video-infra
      receiver: 'dingtalk-video-infra'

inhibit_rules:
  # 网络层故障时抑制上层业务告警，减少噪音
  - source_match:
      alertname: 'NodeNetworkInterfaceDown'
    target_match_re:
      alertname: 'WebRTC.*'
    equal: ['instance', 'cluster']

# auto_remediation/webhook_receiver.py (Flask 简化版)
from flask import Flask, request, jsonify
import subprocess, json, logging

app = Flask(__name__)
logging.basicConfig(level=logging.INFO)

ACTION_MAP = {
    'WebRTCHighPacketLoss': 'restart_media_container',
    'WebRTCHighRTT': 'trigger_qos_downgrade',
    'NodeNetworkInterfaceDown': 'failover_media_node',
}

@app.route('/alert/webhook', methods=['POST'])
def handle_alert():
    payload = request.json
    for alert in payload.get('alerts', []):
        alertname = alert['labels'].get('alertname')
        instance = alert['labels'].get('instance')
        status = alert['status']  # firing / resolved
        
        if status == 'firing' and alertname in ACTION_MAP:
            action = ACTION_MAP[alertname]
            logging.info(f"触发自动化处理: {action} for {instance}")
            execute_action(action, alert['labels'])
    return jsonify({"status": "ok"})

def execute_action(action, labels):
    try:
        if action == 'restart_media_container':
            # 调用 Ansible/SSH 重启容器
            subprocess.run(['ansible', 'media-nodes', '-m', 'docker_container',
                            '-a', f"name=sfu state=restarted"], check=True)
        elif action == 'trigger_qos_downgrade':
            # 调用业务侧降级 API
            subprocess.run(['curl', '-X', 'POST', 
                            f"http://api.example.com/admin/conference/{labels['conference_id']}/downgrade"], check=True)
        elif action == 'failover_media_node':
            # 触发调度器迁移会议
            subprocess.run(['kubectl', 'annotate', 'pod', '-n', 'video-conferencing',
                            '-l', f'node={labels["instance"]}', 'scheduler.alpha.kubernetes.io/evict=true'], check=True)
    except subprocess.CalledProcessError as e:
        logging.error(f"自动化执行失败: {e}")
        # 回落：创建工单或升级通知
        create_incident_ticket(alertname, labels, str(e))

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=8080)

graph TD
    subgraph Edge[边缘/各地域集群]
        PE[Prometheus Edge<br/>保留 2h-6h 热数据]
        PE -->|Remote Write| GW[(Prometheus Remote Write<br/>gRPC/HTTP2 + Snappy 压缩)]
    end
    
    subgraph Central[中心管控平台]
        GW --> TSDB[(长期存储<br/>Thanos/VictoriaMetrics/Cortex)]
        TSDB --> GQ[Global Querier<br/>全局查询入口]
        GQ --> GF[Grafana / 自研大屏]
        GQ --> AL[Alertmanager HA<br/>全局告警去重/路由]
    end
    
    subgraph Meta[元数据中心]
        CMDB[CMDB/服务注册中心] -->|标签注入| PE
        CMDB -->|标签注入| AL
    end

global:
  # 关键：统一外部标签，联邦查询时作为 Sharding Key 与租户隔离依据
  external_labels:
    cluster: 'cn-hangzhou-prod'      # 必须全局唯一
    region: 'cn-east'
    env: 'production'
    team: 'video-infra'

remote_write:
  - url: "https://thanos-receive.central.example.com/api/v1/receive"
    # 生产环境必开：TLS 双向认证 + 基础认证
    tls_config:
      ca_file: /etc/prometheus/certs/ca.crt
      cert_file: /etc/prometheus/certs/client.crt
      key_file: /etc/prometheus/certs/client.key
    basic_auth:
      username: 'edge-writer'
      password_file: /etc/prometheus/secrets/remote_write_pass
    # 性能调优：批量大小、并发、重试
    write_relabel_configs:
      # 核心：剔除高基数标签，仅保留聚合维度，降低中心存储成本 60%+
      - action: labeldrop
        regex: '(pod|container|pod_template_generation|conference_id|participant_id|uuid_.*)'
      - action: labelkeep
        regex: '(cluster|region|env|team|job|instance|service|namespace|node|le)'
    queue_config:
      capacity: 10000        # 内存队列容量
      max_shards: 200        # 并发分片数，匹配中心接收端 CPU 核心数
      min_shards: 50
      max_samples_per_send: 5000
      batch_send_deadline: 5s
      min_backoff: 100ms
      max_backoff: 10s

# 边缘仅保留短周期热数据，降低本地磁盘压力
storage:
  tsdb:
    retention.time: 6h
    retention.size: 10GB
    wal_compression: true

# 1. 安装 Pixie CLI 并部署到目标集群 (需集群管理员权限)
px deploy --cluster_name=video-prod-east --create_artifact_registry=false

# 2. 验证数据采集：进入 Pixie Live UI 或使用 px script 交互式查询
px run px/webrtc_stats  # 内置脚本：自动识别 UDP 端口上的 RTP/RTCP 流

# Pixie Vizier 配置补丁 (ConfigMap: vizier-config)
apiVersion: v1
kind: ConfigMap
metadata:
  name: vizier-config
  namespace: pl
data:
  # 采样率控制：高并发节点建议 10%-20%，避免 CPU 飙升
  sampling_rate: "0.15"  
  # 仅监听媒体服务器监听的 UDP 端口范围，减少无关流量解析
  udp_ports_to_trace: "10000-20000,3478,5349" 
  # 禁用不需要的协议解析器，节省内存
  disabled_protocols: "mysql,postgres,redis,kafka,http2" 
  # 资源限制：防止 eBPF Map 爆内存
  perf_buffer_pages: "128"  # 每 CPU 环形缓冲区页数

# .gitlab-ci.yml / chaos-pipeline.yml
stages:
  - prepare
  - inject
  - observe
  - verify
  - cleanup
  - report

variables:
  CHAOS_NAMESPACE: "video-conferencing"
  TARGET_CLUSTER: "cn-hangzhou-prod"

prepare_environment:
  stage: prepare
  script:
    - kubectl --context=$TARGET_CLUSTER apply -f manifests/chaos-mesh-crds.yaml
    - kubectl --context=$TARGET_CLUSTER label namespace $CHAOS_NAMESPACE chaos-mesh=enabled --overwrite
    # 记录演练前基线指标快照
    - python3 scripts/snapshot_metrics.py --duration=5m --output=baseline_${CI_PIPELINE_ID}.json

inject_network_loss:
  stage: inject
  needs: [prepare_environment]
  script:
    - |
      cat <<EOF | kubectl --context=$TARGET_CLUSTER apply -f -
      apiVersion: chaos-mesh.org/v1alpha1
      kind: NetworkChaos
      metadata:
        name: media-node-packet-loss
        namespace: $CHAOS_NAMESPACE
      spec:
        action: loss
        mode: one
        selector:
          namespaces: [$CHAOS_NAMESPACE]
          labelSelector: "app=media-server"
        loss:
          loss: "15"
          correlation: "25"
        direction: both
        duration: "10m"
      EOF

observe_alert_firing:
  stage: observe
  needs: [inject_network_loss]
  script:
    - sleep 120  # 等待告警评估周期
    - python3 scripts/check_alert.py --alertname=WebRTCHighPacketLoss --status=firing --timeout=300

verify_auto_remediation:
  stage: verify
  needs: [observe_alert_firing]
  script:
    - python3 scripts/verify_downgrade.py --conference_id=$(cat /tmp/affected_conferences.txt)
    - python3 scripts/check_mos_recovery.py --threshold=3.5 --window=5m

cleanup_chaos:
  stage: cleanup
  when: always
  script:
    - kubectl --context=$TARGET_CLUSTER delete networkchaos media-node-packet-loss -n $CHAOS_NAMESPACE --ignore-not-found

generate_report:
  stage: report
  needs: [verify_auto_remediation, cleanup_chaos]
  script:
    - python3 scripts/gen_chaos_report.py 
        --pipeline-id=$CI_PIPELINE_ID 
        --baseline=baseline_${CI_PIPELINE_ID}.json 
        --output=chaos_report_${CI_PIPELINE_ID}.html
  artifacts:
    reports:
      html: chaos_report_${CI_PIPELINE_ID}.html
    expire_in: 90d

# 仅保留聚合维度，剔除 conference_id/participant_id
- record: job:webrtc_packet_loss_ratio:rate5m
  expr: |
    sum by (job, cluster, region, direction) (
      rate(webrtc_inbound_rtp_packets_lost_total[5m])
    )
    /
    sum by (job, cluster, region, direction) (
      rate(webrtc_inbound_rtp_packets_total[5m])
    )
  labels:
    __aggregation__: "true"  # 标记为聚合指标，冷存储仅保留此类