graph LR
A[摄像头/视频流] --> B(MediaPipe Holistic)
B --> C{关键点序列化}
C --> D[WebSocket / WebRTC DataChannel]
D --> E[WebXR 场景]
E --> F[Three.js 骨骼映射 + IK]
F --> G[手语虚拟人渲染]
G --> H[合流推送至会议服务器]

// 初始化 Holistic 模型
const holistic = new Holistic({
  locateFile: (file) => `https://cdn.jsdelivr.net/npm/@mediapipe/holistic/${file}`,
  modelComplexity: 1,
  smoothLandmarks: true,
  minDetectionConfidence: 0.7,
  minTrackingConfidence: 0.7
});

holistic.onResults((results) => {
  const keypoints = {
    // 仅保留手语相关关键点，降低带宽
    leftHand: results.leftHandLandmarks?.map(normalizeLandmark) ?? [],
    rightHand: results.rightHandLandmarks?.map(normalizeLandmark) ?? [],
    face: results.faceLandmarks?.slice(0, 68).map(normalizeLandmark) ?? [], // 仅取面部核心 68 点
    pose: results.poseLandmarks?.slice(0, 25).map(normalizeLandmark) ?? []  // 上半身 25 点
  };
  signalingChannel.send(JSON.stringify({ type: 'keypoints', data: keypoints }));
});

// 归一化至 [-1, 1] 空间，便于后续 IK 映射
function normalizeLandmark(lm) {
  return { x: lm.x * 2 - 1, y: -(lm.y * 2 - 1), z: lm.z * 2, visibility: lm.visibility };
}

// 简易映射示例：MediaPipe 索引 -> Three.js Bone 名称
const HAND_MAP = {
  0: 'wrist',
  1: 'thumb_metacarpal', 2: 'thumb_proximal', 3: 'thumb_distal', 4: 'thumb_tip',
  5: 'index_metacarpal', 6: 'index_proximal', 7: 'index_intermediate', 8: 'index_tip',
  // ... 中指、无名指、小指同理
};

function applyHandPose(handLandmarks, handGroup) {
  handLandmarks.forEach((lm, idx) => {
    const boneName = HAND_MAP[idx];
    if (!boneName) return;
    const bone = handGroup.getObjectByName(boneName);
    if (bone) {
      // 将关键点位置转换为骨骼局部旋转（此处需结合父骨骼做 IK）
      const targetPos = new THREE.Vector3(lm.x, lm.y, lm.z);
      solveIK(bone, targetPos);
    }
  });
}

// 预计算：MediaPipe 468 点 -> ARKit 52 维 BlendShape 系数
const blendshapeSolver = new BlendshapeSolver(mediapipeFaceMesh);

function updateFaceExpression(faceLandmarks) {
  const coeffs = blendshapeSolver.solve(faceLandmarks); // { browInnerUp: 0.8, jawOpen: 0.3, ... }
  Object.entries(coeffs).forEach(([name, value]) => {
    const morphTarget = avatar.mesh.morphTargetDictionary[name];
    if (morphTarget !== undefined) {
      avatar.mesh.morphTargetInfluences[morphTarget] = THREE.MathUtils.lerp(
        avatar.mesh.morphTargetInfluences[morphTarget], value, 0.3 // 平滑插值
      );
    }
  });
}

renderer.xr.enabled = true;
renderer.setAnimationLoop(() => {
  // 1. 更新骨骼动画
  avatar.mixer.update(clock.getDelta());

  // 2. 同步最新关键点数据（双缓冲机制）
  if (keypointBuffer.hasNewFrame()) {
    const kp = keypointBuffer.getLatest();
    applyHandPose(kp.leftHand, avatar.leftHandGroup);
    applyHandPose(kp.rightHand, avatar.rightHandGroup);
    updateFaceExpression(kp.face);
    solveUpperBodyIK(kp.pose); // 肩、肘、躯干 IK
  }

  // 3. XR 帧渲染
  renderer.render(scene, camera);
});

# .github/workflows/deploy.yml
jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install & Build
        run: |
          npm ci
          npm run build:prod   # 输出 dist/ 含 gzip/brotli
      - name: Upload Artifact
        uses: actions/upload-artifact@v4
        with:
          name: webxr-signlang
          path: dist/
  deploy:
    needs: build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/download-artifact@v4
      - name: Deploy to CDN
        run: |
          aws s3 sync dist/ s3://cdn.example.com/webxr-signlang/ --cache-control "public, max-age=31536000, immutable"
          aws cloudfront create-invalidation --distribution-id $CF_ID --paths "/webxr-signlang/*"

// 空间参照点注册表
class SpatialRegistry {
  private referents = new Map<string, THREE.Vector3>(); // 实体 -> 空间坐标
  private nextSlot = 0; // 左侧 -0.4, 中心 0, 右侧 0.4 (米)

  allocate(entity: string): THREE.Vector3 {
    const x = -0.4 + (this.nextSlot % 3) * 0.4;
    const pos = new THREE.Vector3(x, 0, -0.5);
    this.referents.set(entity, pos);
    this.nextSlot++;
    return pos;
  }

  getVerbPath(verb: string, subject: string, object: string): THREE.CurvePath {
    const from = this.referents.get(subject) || new THREE.Vector3();
    const to = this.referents.get(object) || new THREE.Vector3();
    // 生成动词移动轨迹（直线/弧线），供 IK 末端约束跟随
    return new THREE.CurvePath().add(new THREE.LineCurve3(from, to));
  }
}

graph TD
A[基础姿态] --> B[语法姿态 NMS]
B --> C[空间指向 IK]
C --> D[角色扮演偏移]
D --> E[最终骨骼矩阵]

音频流 → VAD → ASR (流式) → 文本规范化 → NMT (中文→Gloss) → NMS/空间标注 → 动画调度 → WebXR渲染

// skinning.wgsl - 顶点着色器中完成蒙皮，Compute Shader 预计算骨骼矩阵
@group(0) @binding(0) var<storage, read> boneMatrices: array<mat4x4<f32>>;
@group(0) @binding(1) var<storage, read> vertexData: array<VertexInput>;
@group(0) @binding(2) var<storage, read_write> outputVertices: array<VertexOutput>;

@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) id: vec3<u32>) {
  let i = id.x;
  if (i >= vertexCount) return;
  
  let v = vertexData[i];
  var pos = vec4<f32>(0.0);
  var nrm = vec3<f32>(0.0);
  
  // 4 骨骼影响
  for (var j = 0; j < 4; j++) {
    let idx = v.joints[j];
    let wt = v.weights[j];
    if (wt == 0.0) continue;
    let m = boneMatrices[idx];
    pos += wt * m * vec4<f32>(v.position, 1.0);
    nrm += wt * (m * vec4<f32>(v.normal, 0.0)).xyz;
  }
  
  outputVertices[i].position = pos.xyz;
  outputVertices[i].normal = normalize(nrm);
}

// skin_brdf.wgsl
fn sss_profile(d: f32) -> vec3<f32> {
  // 基于 Jimenez 2015 可分离 SSS 近似
  let fallback = vec3<f32>(1.0, 0.8, 0.6); // 红色通道穿透最强
  return fallback * exp(-d * vec3<f32>(0.5, 1.2, 2.0));
}

@fragment
fn fs(in: VertexOutput) -> @location(0) vec4<f32> {
  let albedo = textureSample(baseColorTex, samp, in.uv).rgb;
  let normal = unpackNormal(textureSample(normalTex, samp, in.uv));
  let roughness = textureSample(roughnessTex, samp, in.uv).r;
  
  // 屏幕空间 SSS (需深度/法线预通道)
  let sss = screenSpaceSSS(in.screenPos, in.viewDir, normal);
  
  let brdf = cookTorrance(albedo, normal, in.viewDir, roughness, lights);
  return vec4<f32>(brdf + sss * albedo * 0.15, 1.0);
}

# .github/workflows/a11y-test.yml
jobs:
  a11y-regression:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Start Headless Chrome + Xvfb
        run: |
          Xvfb :99 -screen 0 1920x1080x24 &
          export DISPLAY=:99
      - name: Run Puppeteer Test Suite
        run: |
          npm run test:a11y  # 包含：
          # 1. 关键帧对比度自动检测 (axe-core + 自定义规则)
          # 2. IK 收敛性压力测试 (1000 随机姿态, 断言迭代 < 5 次)
          # 3. 端到端延迟基准 (模拟弱网 3G, 断言 P95 < 200ms)
          # 4. 内存泄漏检测 (连续运行 30min, Heap 增长 < 50MB)
      - name: Upload Lighthouse CI Report
        uses: treosh/lighthouse-ci-action@v11