rules/
├── video-conference/
│   ├── infra.rules.yml      # 基础设施层
│   ├── service.rules.yml    # 服务可用性层
│   ├── media.rules.yml      # 媒体质量层
│   └── business.rules.yml   # 业务成功率层

# service.rules.yml
groups:
- name: video-conference-service
  interval: 30s
  rules:
  # P0: 核心信令节点不可用
  - alert: VC_SignalingNodeDown
    expr: up{job=~"jitsi-prosody|mediasoup-signaling", env="prod"} == 0
    for: 1m
    labels:
      severity: critical
      team: vc-platform
      runbook: https://wiki.example.com/runbook/vc-signaling-down
    annotations:
      summary: "视频会议信令节点 {{ $labels.instance }} 不可用"
      description: "节点持续 1 分钟未上报心跳，可能影响会议创建与信令路由。"

  # P1: 会议创建成功率低于 99%
  - alert: VC_CreateConferenceSuccessRateLow
    expr: |
      sum(rate(conference_create_success_total{env="prod"}[5m])) 
      / sum(rate(conference_create_total{env="prod"}[5m])) < 0.99
    for: 3m
    labels:
      severity: warning
      team: vc-platform
    annotations:
      summary: "会议创建成功率低于 99%"
      description: "当前 5 分钟创建成功率为 {{ $value | humanizePercentage }}，请排查信令链路或资源调度模块。"

  # P2: 单节点并发会议数逼近阈值
  - alert: VC_JVBConferenceNearCapacity
    expr: |
      (jvb_conference_active{env="prod"} / jvb_conference_max{env="prod"}) > 0.8
    for: 5m
    labels:
      severity: info
      team: vc-capacity
    annotations:
      summary: "JVB 节点 {{ $labels.instance }} 并发会议占比超 80%"
      description: "当前并发 {{ $value | humanizePercentage }}，建议扩容或触发调度分流。"

global:
  resolve_timeout: 5m
  smtp_smarthost: 'smtp.exmail.qq.com:465'
  smtp_auth_username: 'alert@company.com'
  smtp_auth_password: '${SMTP_PASS}'
  smtp_require_tls: true

templates:
  - '/etc/alertmanager/template/*.tmpl'

route:
  receiver: 'default-webhook'
  group_by: ['alertname', 'cluster', 'service']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  routes:
  # P0 直达电话/短信，绑定值班表
  - match:
      severity: critical
    receiver: 'p0-oncall'
    continue: true
    group_wait: 0s
    repeat_interval: 1h

  # P1 发送企业微信/钉钉群，工作日 09:00-21:00 即时，其余延后
  - match:
      severity: warning
    receiver: 'p1-worktime'
    group_wait: 1m
    repeat_interval: 2h
    # 通过 inhibition_rules 实现夜间静默，见 3.4

  # P2 仅写入归档库/低优先级群，不打扰
  - match:
      severity: info
    receiver: 'p2-archive'
    group_interval: 1h
    repeat_interval: 24h

  # 容量规划团队单独订阅
  - match:
      team: vc-capacity
    receiver: 'capacity-planning'
    continue: true

receivers:
- name: 'p0-oncall'
  webhook_configs:
  - url: 'https://oncall.company.com/api/v1/alerts/prometheus'
    http_config:
      bearer_token: '${ONCALL_TOKEN}'
  wechat_configs:
  - corp_id: '${CORP_ID}'
    api_secret: '${WECHAT_SECRET}'
    to_party: 'oncall-vc'
    send_resolved: true

- name: 'p1-worktime'
  wechat_configs:
  - corp_id: '${CORP_ID}'
    api_secret: '${WECHAT_SECRET}'
    to_party: 'vc-platform'
    send_resolved: true

- name: 'p2-archive'
  webhook_configs:
  - url: 'https://loki.company.com/loki/api/v1/push'

inhibit_rules:
# 节点宕机时，抑制该节点上所有 P1/P2 业务告警
- source_match:
    severity: critical
    alertname: 'VC_SignalingNodeDown'
  target_match_re:
    severity: 'warning|info'
  equal: ['instance', 'cluster']

# 会议创建成功率低时，抑制单节点并发预警（因果关联）
- source_match:
    alertname: 'VC_CreateConferenceSuccessRateLow'
  target_match:
    alertname: 'VC_JVBConferenceNearCapacity'
  equal: ['cluster']

{{ define "wechat.vc.card" }}
{{ range .Alerts }}
**【{{ .Labels.severity | toUpper }}】{{ .Annotations.summary }}**
> **集群**：{{ .Labels.cluster }}
> **实例**：{{ .Labels.instance }}
> **当前值**：{{ .Annotations.description }}
> **触发时间**：{{ .StartsAt.Format "2006-01-02 15:04:05" }}
> **处理指引**：[Runbook]({{ .Labels.runbook }})
> **Grafana**：[跳转面板](https://grafana.company.com/d/vc-overview?var-instance={{ .Labels.instance }})
> **日志查询**：[ELK](https://elk.company.com/app/discover#/?_g=(time:(from:now-30m,to:now))&_a=(query:(query_string:(analyze_wildcard:!t,query:'kubernetes.container_name:%22{{ .Labels.pod }}%22'))))
{{ end }}
{{ end }}

wechat_configs:
- corp_id: '${CORP_ID}'
  api_secret: '${WECHAT_SECRET}'
  to_party: 'vc-platform'
  message: '{{ template "wechat.vc.card" . }}'
  send_resolved: true

# 统一通过 relabel_configs 或 rule 中 labels 注入
labels:
  # 物理拓扑
  region: "cn-hangzhou"        # 必须与 CMDB 资产一致
  zone: "cn-hangzhou-i"
  cluster: "vc-prod-hz-01"     # 唯一集群标识
  # 逻辑拓扑
  env: "prod"
  component: "video-conference"
  tier: "signaling|media|gateway|recording"
  # 业务维度
  team: "vc-platform"
  # 禁止出现高基数标签：pod_ip, container_id, request_id, session_id

# 中央 alertmanager.yml
inhibit_rules:
# 1. AZ 级网络/电源故障：抑制该 AZ 所有子告警
- source_match:
    alertname: 'AZ_Network_Partition'
    severity: 'critical'
  target_match_re:
    severity: 'warning|info'
  equal: ['region', 'zone']  # 同 AZ 内抑制

# 2. 核心依赖挂了：抑制下游业务告警
- source_match:
    alertname: 'VC_Redis_Cluster_Down'
    severity: 'critical'
  target_match_re:
    alertname: 'VC_.*_Timeout|VC_.*_Error_Rate_High'
  equal: ['region', 'cluster']  # 同集群内抑制

package vc.autoremediation

# 默认拒绝
allow = false
reason = "default deny"

# L1: 仅当告警持续 > 5min、且为已知自愈清单、且非人工静默中
allow = true {
    input.alert.labels.severity == "warning"
    input.alert.labels.autoremediate == "l1-safe"
    input.alert.starts_at <= time.now_ns() - 300_000_000_000  # 5min
    not input.alert.annotations.silenced_by_human
    reason = "L1 safe auto-remediation triggered"
}

# L2: 需值班确认标签
allow = true {
    input.alert.labels.severity == "critical"
    input.alert.labels.autoremediate == "l2-stateful"
    input.alert.annotations.approved_by == input.oncall_user
    reason = sprintf("L2 approved by %s", [input.oncall_user])
}

apiVersion: argoproj.io/v1alpha1
kind: WorkflowTemplate
metadata:
  name: vc-jvc-pod-recreate
spec:
  arguments:
    parameters:
    - name: namespace
    - name: podName
  templates:
  - name: main
    steps:
    - - name: cordon-drain
        template: kubectl-cordon-drain
    - - name: delete-pod
        template: kubectl-delete-pod
    - - name: verify-ready
        template: kubectl-wait-ready
  - name: kubectl-cordon-drain
    container:
      image: bitnami/kubectl:latest
      command: [sh, -c]
      args: ["kubectl cordon $(kubectl get pod {{workflow.parameters.podName}} -n {{workflow.parameters.namespace}} -o jsonpath='{.spec.nodeName}') && kubectl drain $(kubectl get pod {{workflow.parameters.podName}} -n {{workflow.parameters.namespace}} -o jsonpath='{.spec.nodeName}') --ignore-daemonsets --delete-emptydir-data --force --grace-period=30"]
  - name: kubectl-delete-pod
    container:
      image: bitnami/kubectl:latest
      command: [sh, -c]
      args: ["kubectl delete pod {{workflow.parameters.podName}} -n {{workflow.parameters.namespace}} --grace-period=10 --force"]
  - name: kubectl-wait-ready
    container:
      image: bitnami/kubectl:latest
      command: [sh, -c]
      args: ["kubectl wait --for=condition=Ready pod -l app=jvb -n {{workflow.parameters.namespace}} --timeout=120s"]

# Workflow 最后一步
- name: verify-and-annotate
  template: http-post-verify
  arguments:
    parameters:
    - name: alert_fingerprint
      value: "{{workflow.parameters.fingerprint}}"
    - name: result
      value: "{{steps.verify-ready.outputs.result}}"

- name: http-post-verify
  container:
    image: curlimages/curl:latest
    command: [sh, -c]
    args: [
      "curl -X POST 'https://alertmanager.company.com/api/v2/alerts/{{workflow.parameters.alert_fingerprint}}/annotations' 
      -H 'Authorization: Bearer $TOKEN' 
      -d '{"autoremediation_result": "{{workflow.parameters.result}}", "verified_at": "$(date -Iseconds)"}'"
    ]

# ❌ 反模式：在规则中实时聚合高基数指标
expr: |
  sum by (cluster, tier) (
    rate(jvb_participants{env="prod"}[5m])
  ) > 1000

# ✅ 正模式：引用预聚合 Recording Rule
# recording rule: vc:participants_rate5m:sum_cluster_tier
# expr: sum by (cluster, tier) (rate(jvb_participants{env="prod"}[5m]))
expr: vc:participants_rate5m:sum_cluster_tier > 1000

// Go 伪代码：上报丢包率时附带 Exemplar
attrs := []attribute.KeyValue{
    attribute.String("conference_id", confID),
    attribute.String("participant_id", partID),
}
exemplar := trace.Exemplar{
    TraceID: spanContext.TraceID(),
    SpanID:  spanContext.SpanID(),
    FilteredAttributes: attrs,
}
meter.Int64Counter("vc_rtp_packets_lost_total").Add(ctx, 1, metric.WithExemplar(exemplar))