# 创建命名空间并开启自动注入
kubectl create namespace meeting
kubectl label namespace meeting istio-injection=enabled

# 验证注入状态
kubectl get namespace meeting --show-labels

Client (Web/App)
    │
    ▼
┌─────────────┐     gRPC/HTTP      ┌──────────────┐
│  API Gateway │ ─────────────────▶ │ Signaling Svc │
└─────────────┘                    └──────┬───────┘
                                           │
                    ┌──────────────────────┼──────────────────────┐
                    ▼                      ▼                      ▼
             ┌─────────────┐        ┌─────────────┐        ┌─────────────┐
             │ Media Relay │        │ Recording   │        │ Transcoding │
             └─────────────┘        └─────────────┘        └─────────────┘

# signaling-v2-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: signaling-v2
  namespace: meeting
  labels:
    app: signaling
    version: v2
spec:
  replicas: 3
  selector:
    matchLabels:
      app: signaling
      version: v2
  template:
    metadata:
      labels:
        app: signaling
        version: v2
    spec:
      containers:
      - name: signaling
        image: registry.example.com/meeting/signaling:v2.1.0
        ports:
        - containerPort: 50051
        resources:
          requests:
            cpu: "500m"
            memory: "512Mi"
          limits:
            cpu: "1000m"
            memory: "1Gi"

kubectl apply -f signaling-v2-deployment.yaml
kubectl rollout status deployment/signaling-v2 -n meeting

# virtualservice-signaling-mirror.yaml
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: signaling
  namespace: meeting
spec:
  hosts:
  - signaling
  http:
  - match:
    - headers:
        x-canary:
          exact: "true"   # 可选：仅镜像带标记的请求
    route:
    - destination:
        host: signaling
        subset: v1
      weight: 100
    mirror:
      host: signaling
      subset: v2
    mirrorPercentage:
      value: 10.0  # 镜像 10% 流量
---
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: signaling
  namespace: meeting
spec:
  host: signaling
  trafficPolicy:
    connectionPool:
      tcp:
        maxConnections: 1000
      http:
        h2UpgradePolicy: UPGRADE
        http1MaxPendingRequests: 1000
        http2MaxRequests: 1000
  subsets:
  - name: v1
    labels:
      version: v1
  - name: v2
    labels:
      version: v2

kubectl apply -f virtualservice-signaling-mirror.yaml

# fault-injection-media-relay-delay.yaml
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: media-relay
  namespace: meeting
spec:
  hosts:
  - media-relay
  http:
  - fault:
      delay:
        percentage:
          value: 100.0
        fixedDelay: 200ms
    route:
    - destination:
        host: media-relay
        subset: v1
      weight: 100

# fault-injection-recording-abort.yaml
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: recording
  namespace: meeting
spec:
  hosts:
  - recording
  http:
  - fault:
      abort:
        percentage:
          value: 5.0
        httpStatus: 503
    route:
    - destination:
        host: recording
        subset: v1
      weight: 100

# fault-injection-transcoding-grpc-abort.yaml
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: transcoding
  namespace: meeting
spec:
  hosts:
  - transcoding
  http:
  - match:
    - headers:
        content-type:
          prefix: "application/grpc"
    fault:
      abort:
        percentage:
          value: 3.0
        grpcStatus: UNAVAILABLE
    route:
    - destination:
        host: transcoding
        subset: v1
      weight: 100

# destinationrule-transcoding-outlier.yaml
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: transcoding
  namespace: meeting
spec:
  host: transcoding
  trafficPolicy:
    connectionPool:
      tcp:
        maxConnections: 500
      http:
        h2UpgradePolicy: UPGRADE
        http2MaxRequests: 1000
        maxRequestsPerConnection: 100
    outlierDetection:
      consecutive5xxErrors: 3
      interval: 10s
      baseEjectionTime: 30s
      maxEjectionPercent: 50
      minHealthPercent: 30
  subsets:
  - name: v1
    labels:
      version: v1

# combined-mirror-fault.yaml
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: signaling
  namespace: meeting
spec:
  hosts:
  - signaling
  http:
  - route:
    - destination:
        host: signaling
        subset: v1
      weight: 100
    mirror:
      host: signaling
      subset: v2
    mirrorPercentage:
      value: 10.0
    # 仅对镜像流量注入故障（需 Istio 1.18+ 支持 mirrorFault）
    mirrorFault:
      delay:
        percentage:
          value: 100.0
        fixedDelay: 100ms

# 故障注入演练复盘报告 - YYYYMMDD

## 1. 演练基本信息
- 时间窗口：2024-01-15 20:00 ~ 21:30
- 参与角色：SRE、后端开发、QA、产品经理
- 影响范围：仅镜像流量 / 仅测试租户

## 2. 执行清单与结果
| 场景 | 预期结果 | 实际结果 | 偏差分析 | 整改措施 | 责任人 | 截止日期 |
|------|----------|----------|----------|----------|--------|----------|
| media-relay 200ms 延迟 | 客户端 3s 内重连成功 | 5% 客户端超时退出 | 客户端重试间隔固定 1s，未指数退避 | 修改 SDK 重试策略 | @client-team | 2024-01-22 |

## 3. 监控盲区补充
- 新增：`signaling_v2_ice_candidate_collection_duration_seconds` Histogram
- 新增：`recording_fallback_queue_length` Gauge

## 4. 后续计划
- 将故障注入纳入 CI/CD 流水线（每周自动化演练）
- 引入 Chaos Mesh 进行基础设施层故障演练（节点断电、磁盘满等）

# virtualservice-timeout-retry.yaml
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: signaling
  namespace: meeting
spec:
  hosts:
  - signaling
  http:
  # 1. 核心信令：严格超时，禁用重试
  - match:
    - headers:
        x-service-type:
          exact: "core-signaling"
    route:
    - destination:
        host: signaling
        subset: v1
      weight: 100
    timeout: 1.5s
    retries:
      attempts: 0  # 显式禁用
    fault:  # 配合故障注入验证超时降级
      abort:
        percentage:
          value: 1.0
        httpStatus: 504

  # 2. 辅助服务：宽松超时，激进重试
  - match:
    - headers:
        x-service-type:
          exact: "auxiliary"
    route:
    - destination:
        host: recording
        subset: v1
      weight: 100
    timeout: 5s
    retries:
      attempts: 3
      perTryTimeout: 1.5s
      backoff:
        baseInterval: 0.1s
        multiplier: 2.0  # 指数退避: 100ms, 200ms, 400ms
      retryOn: "connect-failure,refused-stream,unavailable,cancelled,retriable-status-codes"
      retryRemoteLocalities: true  # 允许跨可用区重试

# envoyfilter-tc-netem.yaml
apiVersion: networking.istio.io/v1beta1
kind: EnvoyFilter
metadata:
  name: media-relay-netem-inject
  namespace: meeting
spec:
  workloadSelector:
    labels:
      app: media-relay
  configPatches:
  - applyTo: NETWORK_FILTER
    match:
      context: SIDECAR_INBOUND
      listener:
        filterChain:
          filter:
            name: envoy.filters.network.tcp_proxy
    patch:
      operation: INSERT_BEFORE
      value:
        name: envoy.filters.network.lua
        typed_config:
          "@type": type.googleapis.com/envoy.extensions.filters.network.lua.v3.Lua
          inline_code: |
            function envoy_on_request(request_handle)
              -- 仅在演练标记开启时注入 tc 规则 (通过 Header 或 Metadata 触发)
              if request_handle:headers():get("x-chaos-netem") == "enable" then
                -- 使用 os.execute 调用 tc (需 Sidecar 具备 NET_ADMIN CAP)
                -- 示例: 丢包 2%, 延迟 50ms±10ms, 重复 1%
                os.execute("tc qdisc add dev eth0 root netem loss 2% delay 50ms 10ms duplicate 1% 2>/dev/null || tc qdisc change dev eth0 root netem loss 2% delay 50ms 10ms duplicate 1%")
                request_handle:logInfo("Netem injected: loss 2%, delay 50ms±10ms")
              end
            end

# values-istio-operator.yaml 片段
sidecarInjectorWebhook:
  templates:
    default:
      patches:
      - patch: |
          - op: add
            path: /spec/template/spec/containers/0/securityContext/capabilities/add
            value: ["NET_ADMIN"]
          - op: add
            path: /spec/template/spec/containers/0/securityContext/privileged
            value: true  # 简化方案，生产建议仅加 CAP_NET_ADMIN + CAP_SYS_ADMIN

# 仅对带标记的测试流量生效，不影响生产
curl -H "x-chaos-netem: enable" https://meeting.example.com/api/join

# networkchaos-media-relay.yaml
apiVersion: chaos-mesh.org/v1alpha1
kind: NetworkChaos
metadata:
  name: media-relay-weak-network
  namespace: meeting
spec:
  action: loss  # 支持: delay, loss, duplicate, corrupt, partition, bandwidth
  mode: one  # one / all / fixed / fixed-percent / random-max-percent
  selector:
    namespaces:
      - meeting
    labelSelectors:
      app: media-relay
  loss:
    loss: "2"      # 丢包 2%
    correlation: "10"  # 相关性 10% (模拟突发丢包)
  delay:
    latency: "50ms"
    correlation: "20"
    jitter: "10ms"
  duration: "300s"  # 持续 5 分钟
  scheduler:
    cron: "@every 1h"  # 定时自动化演练

# rollout-signaling.yaml
apiVersion: argoproj.io/v1alpha1
kind: Rollout
metadata:
  name: signaling
  namespace: meeting
spec:
  replicas: 10
  strategy:
    canary:
      canaryMetadata:
        labels:
          version: canary
      stableMetadata:
        labels:
          version: stable
      steps:
      - setWeight: 5
      - pause: {duration: 10m}  # 观测窗口
      - setWeight: 20
      - pause: {duration: 10m}
      - setWeight: 50
      - pause: {duration: 10m}
      # 关键：引入分析模板自动化判定
      analysis:
        templates:
        - templateName: signaling-canary-analysis
        args:
        - name: service-name
          value: signaling
        - name: namespace
          value: meeting
  selector:
    matchLabels:
      app: signaling
  template:
    metadata:
      labels:
        app: signaling
    spec:
      containers:
      - name: signaling
        image: registry.example.com/meeting/signaling:v2.1.0

# analysis-template-signaling.yaml
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: signaling-canary-analysis
  namespace: meeting
spec:
  args:
  - name: service-name
  - name: namespace
  metrics:
  - name: success-rate
    interval: 1m
    count: 10  # 连续 10 分钟
    successCondition: result[0] >= 0.995
    failureLimit: 3
    provider:
      prometheus:
        address: http://prometheus.monitoring.svc:9090
        query: |
          sum(rate(istio_requests_total{destination_service="{{args.service-name}}",destination_version="canary",response_code=~"2.."}[1m]))
          /
          sum(rate(istio_requests_total{destination_service="{{args.service-name}}",destination_version="canary"}[1m]))
  - name: p99-latency
    interval: 1m
    count: 10
    successCondition: result[0] <= 300  # 单位 ms
    provider:
      prometheus:
        address: http://prometheus.monitoring.svc:9090
        query: |
          histogram_quantile(0.99, sum(rate(istio_request_duration_milliseconds_bucket{destination_service="{{args.service-name}}",destination_version="canary"}[1m])) by (le))
  - name: error-budget-burn-rate
    interval: 5m
    count: 1
    successCondition: result[0] < 2  # 燃烧率 < 2x
    provider:
      prometheus:
        address: http://prometheus.monitoring.svc:9090
        query: |
          (sum(rate(istio_requests_total{destination_service="{{args.service-name}}",destination_version="canary",response_code=~"5.."}[5m])) / sum(rate(istio_requests_total{destination_service="{{args.service-name}}",destination_version="canary"}[5m])))
          /
          (sum(rate(istio_requests_total{destination_service="{{args.service-name}}",destination_version="stable",response_code=~"5.."}[1h])) / sum(rate(istio_requests_total{destination_service="{{args.service-name}}",destination_version="stable"}[1h])))

# 预发布阶段注入 100ms 延迟 + 1% 503 错误
- analysis:
    templates:
    - templateName: signaling-resilience-test
    args:
    - name: fault-duration
      value: "5m"
---
apiVersion: argoproj.io/v1alpha1
kind: AnalysisTemplate
metadata:
  name: signaling-resilience-test
spec:
  args:
  - name: fault-duration
  metrics:
  - name: inject-fault
    # 使用 Argo Rollouts 的 Job 类型指标，启动一个 Job 来下发 VirtualService 故障规则
    provider:
      job:
        spec:
          template:
            spec:
              serviceAccountName: argo-rollouts-istio-manager
              containers:
              - name: inject-fault
                image: bitnami/kubectl:latest
                command: ["/bin/sh", "-c"]
                args:
                - |
                  cat <<EOF | kubectl apply -f -
                  apiVersion: networking.istio.io/v1beta1
                  kind: VirtualService
                  metadata:
                    name: signaling-canary-fault
                    namespace: meeting
                  spec:
                    hosts: [signaling]
                    http:
                    - match:
                      - headers:
                          x-canary: "true"
                      fault:
                        delay:
                          percentage: {value: 100.0}
                          fixedDelay: 100ms
                        abort:
                          percentage: {value: 1.0}
                          httpStatus: 503
                      route:
                      - destination:
                          host: signaling
                          subset: canary
                  EOF
                  sleep {{args.fault-duration}}
                  kubectl delete virtualservice signaling-canary-fault -n meeting
  - name: verify-resilience
    interval: 30s
    count: 10
    successCondition: result[0] >= 0.99  # 故障注入下成功率仍 > 99%
    provider:
      prometheus:
        query: |
          sum(rate(istio_requests_total{destination_service="signaling",destination_version="canary",response_code=~"2.."}[30s]))
          /
          sum(rate(istio_requests_total{destination_service="signaling",destination_version="canary"}[30s]))

# virtualservice-cross-cluster-mirror.yaml
apiVersion: networking.istio.io/v1beta1
kind: VirtualService
metadata:
  name: signaling
  namespace: meeting
spec:
  hosts:
  - signaling.meeting.svc.cluster.local  # 本地服务 FQDN
  http:
  - route:
    - destination:
        host: signaling.meeting.svc.cluster.local
        subset: v1-huadong
      weight: 100
    mirror:
      host: signaling.meeting.svc.cluster.local
      subset: v2-singapore  # 远程集群 Subset (需 DestinationRule 定义)
    mirrorPercentage:
      value: 5.0  # 跨地域镜像建议低比例，控制带宽成本
---
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: signaling
  namespace: meeting
spec:
  host: signaling.meeting.svc.cluster.local
  trafficPolicy:
    tls:
      mode: ISTIO_MUTUAL
  subsets:
  - name: v1-huadong
    labels:
      version: v1
      topology.istio.io/region: huadong
  - name: v2-singapore
    labels:
      version: v2
      topology.istio.io/region: singapore

# 模拟 华东 -> 新加坡 链路 5% 丢包、200ms 抖动
# 需在 华东集群 的 Sidecar 出站方向注入 (EnvoyFilter + tc 或 ChaosMesh)
# 目标 IP 段为 新加坡集群 Pod CIDR / Service CIDR

# sidecar-resource-profile.yaml
apiVersion: networking.istio.io/v1beta1
kind: Sidecar
metadata:
  name: media-relay-sidecar
  namespace: meeting
spec:
  workloadSelector:
    labels:
      app: media-relay
  egress:
  - hosts:
    - "./*"
    - "istio-system/*"
  # 资源限制建议通过 Deployment Patch 或 Mutation Webhook 注入
  # 此处仅示意配置结构

# destinationrule-mirror-protection.yaml
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: signaling-v2-mirror-protection
  namespace: meeting
spec:
  host: signaling
  subsets:
  - name: v2
    labels:
      version: v2
    trafficPolicy:
      connectionPool:
        tcp:
          maxConnections: 500  # 限制镜像流量连接数
        http:
          h2UpgradePolicy: UPGRADE
          http1MaxPendingRequests: 200
          http2MaxRequests: 500
          maxRequestsPerConnection: 50
      outlierDetection:
        consecutive5xxErrors: 2
        interval: 5s
        baseEjectionTime: 10s
        maxEjectionPercent: 80  # 允许大比例剔除，保护主流量

# Sidecar 因镜像流量新增 CPU 使用率 (近似)
(
  sum(rate(container_cpu_usage_seconds_total{namespace="meeting", container="istio-proxy", pod=~".*-v2"}[5m]))
  /
  sum(kube_pod_container_resource_requests{namespace="meeting", container="istio-proxy", resource="cpu", pod=~".*-v2"})
) * 100

# 镜像流量带宽成本估算 (需结合云厂商单价)
sum(rate(container_network_transmit_bytes_total{namespace="meeting", pod=~".*-v2"}[5m])) * 8 / 1024 / 1024  # Mbps

# authorizationpolicy-allow-mirror.yaml
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
  name: signaling-v2-allow-mirror
  namespace: meeting
spec:
  selector:
    matchLabels:
      app: signaling
      version: v2
  action: ALLOW
  rules:
  - from:
    - source:
        principals: ["cluster.local/ns/meeting/sa/signaling"]  # 来源主流量 Sidecar 身份
    - source:
        principals: ["cluster.local/ns/istio-system/sa/istio-ingressgateway"]  # 网关
  - to:
    - operation:
        methods: ["GET", "POST", "GRPC"]
        paths: ["/join", "/leave", "/candidate/*"]

# 仅在演练命名空间/标签下生效
apiVersion: security.istio.io/v1beta1
kind: AuthorizationPolicy
metadata:
  name: signaling-fault-injection-bypass
  namespace: meeting
spec:
  selector:
    matchLabels:
      app: signaling
  action: ALLOW
  rules:
  - from:
    - source:
        requestPrincipals: ["*"]  # 允许所有来源 (仅演练窗口开启)
    when:
    - key: request.headers[x-chaos-test]
      values: ["true"]

#!/bin/bash
# cleanup-istio-drill.sh
# 用途: 演练结束后一键清理所有镜像/故障注入相关 Istio 资源
# 使用: ./cleanup-istio-drill.sh <namespace> [label-selector]

NS=${1:-meeting}
SELECTOR=${2:-"drill-type in (mirror,fault-injection,netem)"}

echo ">>> Cleaning VirtualServices in $NS ..."
kubectl get virtualservice -n $NS -l "$SELECTOR" -o name | xargs -r kubectl delete -n $NS

echo ">>> Cleaning DestinationRules in $NS ..."
kubectl get destinationrule -n $NS -l "$SELECTOR" -o name | xargs -r kubectl delete -n $NS

echo ">>> Cleaning EnvoyFilters in $NS ..."
kubectl get envoyfilter -n $NS -l "$SELECTOR" -o name | xargs -r kubectl delete -n $NS

echo ">>> Cleaning NetworkChaos (ChaosMesh) in $NS ..."
kubectl get networkchaos -n $NS -l "$SELECTOR" -o name | xargs -r kubectl delete -n $NS

echo ">>> Cleaning AuthorizationPolicies (drill) in $NS ..."
kubectl get authorizationpolicy -n $NS -l "drill-type=auth-bypass" -o name | xargs -r kubectl delete -n $NS

echo ">>> Verifying cleanup ..."
kubectl get virtualservice,destinationrule,envoyfilter,networkchaos,authorizationpolicy -n $NS -l "$SELECTOR" 2>/dev/null || echo "All clean."

echo ">>> Restarting affected sidecars to flush tc rules (if any) ..."
# 仅重启打了 drill-sidecar-restart=true 标签的 Pod
kubectl rollout restart deployment -n $NS -l drill-sidecar-restart=true

echo "Done."