graph LR
    A[媒体节点 Pod] -->|gRPC Stream| B(指标聚合 Sidecar)
    B -->|Prometheus 格式| C[Pushgateway]
    C -->|Pull| D[Prometheus Server]
    D -->|Metrics API| E[K8s HPA Controller]

func (s *Sidecar) PushLoop(ctx context.Context) {
    ticker := time.NewTicker(15 * time.Second)
    for {
        select {
        case <-ctx.Done(): return
        case <-ticker.C:
            metrics := s.collectFromLocalGRPC() // 连接本地媒体进程
            if err := s.pushToGateway(metrics); err != nil {
                log.Errorw("push failed", "err", err)
            }
        }
    }
}

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: sfu-media-hpa
  namespace: meeting
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: StatefulSet   # 媒体节点建议用 StatefulSet 保持网络标识稳定
    name: sfu-media
  minReplicas: 3
  maxReplicas: 200
  behavior:
    scaleDown:
      stabilizationWindowSeconds: 600   # 缩容冷却 10 分钟，防抖
      policies:
      - type: Percent
        value: 10
        periodSeconds: 300
    scaleUp:
      stabilizationWindowSeconds: 60    # 扩容响应 1 分钟
      policies:
      - type: Percent
        value: 50
        periodSeconds: 60
      - type: Pods
        value: 10
        periodSeconds: 60
      selectPolicy: Max
  metrics:
  - type: Pods
    pods:
      metric:
        name: active_conferences
      target:
        type: AverageValue
        averageValue: "25"   # 单 Pod 目标承载 25 个会议
  - type: Pods
    pods:
      metric:
        name: concurrent_streams
      target:
        type: AverageValue
        averageValue: "150"
  - type: External
    external:
      metric:
        name: avg_packet_loss_rate
        selector:
          matchLabels:
            cluster: "prod"
      target:
        type: Value
        value: "0.003"   # 0.3% 丢包率触发扩容

apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
  name: sfu-media-vpa
  namespace: meeting
spec:
  targetRef:
    apiVersion: apps/v1
    kind: StatefulSet
    name: sfu-media
  updatePolicy:
    updateMode: "Off"   # 仅给出建议，人工或 GitOps 同步到 StatefulSet
  resourcePolicy:
    containerPolicies:
    - containerName: sfu
      controlledResources: ["cpu", "memory"]
      minAllowed:
        cpu: "500m"
        memory: "2Gi"
      maxAllowed:
        cpu: "8"
        memory: "16Gi"

func (p *MediaSchedulerPlugin) Score(ctx context.Context, state *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) {
    nodeInfo, _ := p.handle.SnapshotSharedLister().NodeInfos().Get(nodeName)
    var score int64

    // 1. 同 Zone 优先
    if pod.Labels["topology.kubernetes.io/zone"] == nodeInfo.Node().Labels["topology.kubernetes.io/zone"] {
        score += 100
    }

    // 2. GPU 显存剩余比例加权
    allocatableGPU := nodeInfo.Allocatable.Resource(v1.ResourceName("nvidia.com/gpu"))
    requestedGPU := nodeInfo.Requested.Resource(v1.ResourceName("nvidia.com/gpu"))
    if allocatableGPU > 0 {
        score += int64(50 * (float64(allocatableGPU-requestedGPU) / float64(allocatableGPU)))
    }

    // 3. 网络带宽剩余
    if bw, ok := nodeInfo.Node().Annotations["node.beta.kubernetes.io/available-bandwidth-mbps"]; ok {
        if val, _ := strconv.Atoi(bw); val > 1000 {
            score += 30
        }
    }

    return score, nil
}

sequenceDiagram
    participant HPA
    participant Drain-Controller
    participant Media-Pod
    participant Signaling
    HPA->>Drain-Controller: 触发缩容候选 Pod 列表
    Drain-Controller->>Media-Pod: 打标签 `draining=true` + 停止新建会议
    Media-Pod->>Signaling: 通知现有会议“即将迁移”
    loop 每 5 秒轮询
        Media-Pod-->>Drain-Controller: 上报 active_sessions=0
    end
    Drain-Controller->>K8s API: Delete Pod (gracePeriod=120s)

# pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: sfu-media-pdb
  namespace: meeting
spec:
  minAvailable: 80%   # 保证至少 80% 节点可用
  selector:
    matchLabels:
      app: sfu-media

# statefulset 片段
lifecycle:
  preStop:
    exec:
      command: ["/bin/sh","-c","/usr/local/bin/graceful_shutdown.sh"]

# cluster-autoscaler 启动参数片段
--scale-down-utilization-threshold=0.4
--scale-down-unneeded-time=10m
--scale-down-unready-time=5m
--skip-nodes-with-system-pods=false
--skip-nodes-with-local-storage=false
--node-group-auto-discovery=asg:tag=k8s.io/cluster-autoscaler/enabled,k8s.io/cluster-autoscaler/meeting-prod

resource "aws_autoscaling_group" "spot_gpu" {
  name_prefix = "meeting-spot-gpu-"
  mixed_instances_policy {
    launch_template {
      launch_template_specification = { ... }
      override {
        instance_type = "g5.xlarge"
        weighted_capacity = "1"
      }
      override {
        instance_type = "g4dn.xlarge"
        weighted_capacity = "1"
      }
    }
    instances_distribution {
      spot_allocation_strategy = "capacity-optimized-prioritized"
      on_demand_base_capacity  = 0
      on_demand_percentage_above_base_capacity = 0
    }
  }
  tag {
    key                 = "k8s.io/cluster-autoscaler/node-pool"
    value               = "spot-gpu"
    propagate_at_launch = true
  }
  tag {
    key                 = "meeting.io/tier"
    value               = "burst"
    propagate_at_launch = true
  }
}

# 媒体节点 PodSpec 片段
tolerations:
- key: "node.kubernetes.io/spot"
  operator: "Exists"
  effect: "NoSchedule"
affinity:
  nodeAffinity:
    preferredDuringSchedulingIgnoredDuringExecution:
    - weight: 80
      preference:
        matchExpressions:
        - key: meeting.io/tier
          operator: In
          values: ["base", "elastic"]   # 优先非 Spot
    - weight: 20
      preference:
        matchExpressions:
        - key: meeting.io/tier
          operator: In
          values: ["burst"]

{
  "timestamp": "2024-05-20T10:15:00.123Z",
  "level": "INFO",
  "component": "sfu-media",
  "pod": "sfu-media-7",
  "session_id": "conf_abc123",
  "action": "migrate_start",
  "target_pod": "sfu-media-12",
  "reason": "scale_down",
  "duration_ms": 420
}

# CRD 定义片段
apiVersion: meeting.io/v1alpha1
kind: MeetingCluster
metadata:
  name: prod-cluster
spec:
  signaling:
    replicas: 6
    hpaRef: signaling-hpa
  media:
    minReplicas: 10
    maxReplicas: 500
    hpaRef: sfu-media-hpa
    schedulerName: media-scheduler
  coordination:
    ratio: "1:50"           # 信令:媒体 容量比参考值
    scaleUpDelaySeconds: 30 # 信令先扩容，30s后允许媒体扩容
    scaleDownDelaySeconds: 300 # 媒体先缩容，5分钟后允许信令缩容

func (r *MeetingClusterReconciler) reconcileCoordination(ctx context.Context, mc *MeetingCluster) error {
    // 1. 获取当前实际副本数
    sigReady := getReadyReplicas(mc.Spec.Signaling.HPARef)
    mediaReady := getReadyReplicas(mc.Spec.Media.HPARef)

    // 2. 计算目标比例
    targetRatio := parseRatio(mc.Spec.Coordination.Ratio) // 1:50
    desiredMedia := sigReady * targetRatio

    // 3. 扩容协调：信令领跑
    if desiredMedia > mediaReady {
        // 先扩信令，等信令 Ready 后再放开媒体 HPA minReplicas
        if sigReady >= mc.Spec.Signaling.Replicas {
            updateHPAMinReplicas(mc.Spec.Media.HPARef, desiredMedia)
        }
    }

    // 4. 缩容协调：媒体领跑
    if desiredMedia < mediaReady {
        // 先降媒体 HPA maxReplicas，待媒体缩容完成后再降信令
        if mediaReady <= desiredMedia + buffer {
            updateHPAMaxReplicas(mc.Spec.Signaling.HPARef, sigReady - 1)
        }
    }
    return nil
}

helm repo add kedacore https://kedacore.github.io/charts
helm install keda kedacore/keda --namespace keda --create-namespace 
  --set prometheus.metricServer.enabled=true

# predictor.py 核心逻辑
def calculate_desired_replicas(now: datetime) -> int:
    base = get_historical_baseline(now)           # 历史同期基线
    scheduled = get_scheduled_conferences(now, window_min=15) # 日历会议
    trend = get_realtime_trend()                  # 近 5 分钟增长斜率
    
    # 启发式公式：基线 + 预约峰值 * 安全系数 + 趋势修正
    predicted = base + scheduled * 1.3 + trend * 300
    return max(min(predicted, MAX_REPLICAS), MIN_REPLICAS)

# 推送指标
push_to_gateway(
    "pushgateway:9091", 
    job="meeting-predictor", 
    grouping_dict={"cluster": "prod"},
    registry=registry
)
# 指标名：meeting_predicted_media_replicas

apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: sfu-media-keda
  namespace: meeting
spec:
  scaleTargetRef:
    name: sfu-media
  pollingInterval: 30
  cooldownPeriod: 600
  minReplicaCount: 10
  maxReplicaCount: 500
  advanced:
    restoreToOriginalReplicaCount: false
    horizontalPodAutoscalerConfig:
      behavior:
        scaleUp:
          stabilizationWindowSeconds: 30
  triggers:
  - type: prometheus
    metadata:
      serverAddress: http://prometheus.monitoring:9090
      metricName: meeting_predicted_media_replicas
      query: meeting_predicted_media_replicas{cluster="prod"}
      threshold: "1" # 只要有值就触发
      activateQuery: 'meeting_predicted_media_replicas{cluster="prod"} > 10'
  - type: cpu # 保留 CPU 作为兜底
    metadata:
      type: Utilization
      value: "70"

# 媒体节点 PodSpec
spec:
  containers:
  - name: sfu
    resources:
      limits:
        cpu: "16"
        memory: "32Gi"
        nvidia.com/gpu: "1"
        media.io/gpu-encoder: "2"      # 独占 2 个 NVENC 实例
        media.io/smartnic-vf: "1"      # 独占 1 个 VF
    env:
    - name: GOMEMLIMIT
      value: "28Gi"
  # 关键：拓扑对齐，保证 CPU/内存/GPU/网卡在同一 NUMA Node
  topologySpreadConstraints:
  - maxSkew: 1
    topologyKey: topology.kubernetes.io/numa-node
    whenUnsatisfiable: DoNotSchedule
    labelSelector:
      matchLabels:
        app: sfu-media

func numaAlignmentScore(node *v1.Node, pod *v1.Pod) int64 {
    // 解析 Pod 请求的拓扑资源
    requestedNUMAs := getRequestedNUMANodes(pod) // 从 Device Plugin 分配结果反推
    if len(requestedNUMAs) == 0 {
        return 50 // 无拓扑要求，中性分
    }
    
    // 检查节点当前可分配的 NUMA 资源是否满足“同 NUMA”约束
    allocatable := getNodeNUMACapacity(node)
    for _, numaID := range requestedNUMAs {
        if allocatable[numaID].CPU >= podRequestCPU && 
           allocatable[numaID].Memory >= podRequestMem {
            return 100 // 完美对齐
        }
    }
    return 0 // 跨 NUMA，性能损耗大，不调度
}

graph TB
    subgraph ControlPlane [控制平面 - 核心集群]
        Karmada[Karmada Control Plane]
        GlobalHPA[Global HPA / MeetingCluster CR]
        Scheduler[联邦调度器]
    end
    
    subgraph EdgeCN [华东边缘集群]
        EdgeKube[Kubelet + Media Pods]
        LocalHPA[Local HPA]
    end
    
    subgraph EdgeUS [美西边缘集群]
        EdgeKube2[...]
    end

    GlobalHPA -->|下发期望副本数| Scheduler
    Scheduler -->|PropagationPolicy| EdgeCN
    Scheduler -->|PropagationPolicy| EdgeUS
    LocalHPA -->|实时微调| EdgeKube

# 边缘节点池配置差异化
apiVersion: cluster.karmada.io/v1alpha1
kind: Cluster
metadata:
  name: edge-shanghai-01
  labels:
    region: cn-east
    tier: edge
    network-type: "5g-dedicated" # 专线/5G专网
spec:
  syncMode: Push
  # 边缘节点资源少，设置更保守的驱逐阈值
  taints:
  - key: "node.kubernetes.io/edge"
    effect: "NoSchedule"

tolerations:
- key: "node.kubernetes.io/edge"
  operator: "Exists"
  effect: "NoSchedule"
- key: "node.kubernetes.io/unreachable"
  operator: "Exists"
  effect: "NoExecute"
  tolerationSeconds: 30 # 边缘网络抖动容忍 30s，避免误驱逐

infra-config/
├── apps/
│   ├── meeting/
│   │   ├── base/                 # 基础资源：StatefulSet, Service, PDB
│   │   │   ├── statefulset.yaml
│   │   │   └── kustomization.yaml
│   │   ├── overlays/
│   │   │   ├── dev/              # 开发环境：副本少、资源小、无GPU
│   │   │   │   ├── kustomization.yaml
│   │   │   │   └── hpa-patch.yaml
│   │   │   ├── staging/          # 预发环境：镜像生产配置、压测开关
│   │   │   └── prod/             # 生产环境：多节点池、完整监控
│   │   │       ├── kustomization.yaml
│   │   │       ├── hpa-prod.yaml
│   │   │       ├── vpa-prod.yaml
│   │   │       ├── scheduler-config.yaml
│   │   │       └── network-policy.yaml
│   │   └── keda/
│   │       └── scaledobject.yaml
├── clusters/
│   ├── prod-shanghai/
│   │   └── applications.yaml     # ArgoCD Application 定义
│   └── prod-beijing/
└── policies/                     # 策略即代码
    ├── kyverno/
    │   ├── require-signed-image.yaml
    │   └── restrict-privileged.yaml
    └── cost/
        └── limitrange-media.yaml

apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: meeting-media-prod
spec:
  generators:
  - clusters:
      selector:
        matchLabels:
          environment: prod
          tier: media-node
  template:
    metadata:
      name: 'meeting-media-{{name}}'
    spec:
      project: meeting
      source:
        repoURL: https://git.company.com/infra-config.git
        targetRevision: HEAD
        path: apps/meeting/overlays/prod
        kustomize:
          commonLabels:
            cluster: '{{name}}'
      destination:
        server: '{{server}}'
        namespace: meeting
      syncPolicy:
        automated:
          prune: true
          selfHeal: true
          allowEmpty: false
        syncOptions:
        - Validate=true
        - CreateNamespace=true
        retry:
          limit: 5
          backoff:
            duration: 30s
            factor: 2
            maxDuration: 5m

# 单会议边际成本 (实时)
sum(rate(container_cpu_usage_seconds_total{namespace="meeting",pod=~"sfu-media-.*"}[5m])) by (pod) 
/ 
sum(active_conferences{pod=~"sfu-media-.*"}) by (pod)
* 
on (pod) group_left(node) 
node_hourly_cost{cluster="prod"}

# 弹性节省率 (周报)
(1 - (sum(spot_instance_cost) + sum(on_demand_elastic_cost)) / sum(reserved_equivalent_cost)) * 100

# 伸缩响应成本比
sum(increase(hpa_scale_up_events_total[1h])) / sum(increase(meeting_joined_total[1h]))